diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 12a03d0..1570fbf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -224,8 +224,8 @@ public static final String HIVE_ADDED_JARS = "hive.added.jars"; public static final String VECTOR_MODE = "VECTOR_MODE"; public static final String USE_VECTORIZED_INPUT_FILE_FORMAT = "USE_VECTORIZED_INPUT_FILE_FORMAT"; - public static String MAPNAME = "Map "; - public static String REDUCENAME = "Reducer "; + public static String MAPNAME = "Vertex "; + public static String REDUCENAME = "Vertex "; /** * ReduceField: diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index aa2dfc7..eadcf93 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -273,7 +273,7 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, case BROADCAST_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; - case CUSTOM_EDGE: { + case CO_PARTITION_EDGE: { mergeInputClass = ConcatenatedMergedKeyValueInput.class; int numBuckets = edgeProp.getNumBuckets(); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); @@ -288,11 +288,11 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, break; } - case CUSTOM_SIMPLE_EDGE: + case PARTITION_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; - case SIMPLE_EDGE: + case SORT_PARTITION_EDGE: setupAutoReducerParallelism(edgeProp, w); // fall through @@ -319,7 +319,7 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr throws IOException { switch(edgeProp.getEdgeType()) { - case CUSTOM_EDGE: { + case CO_PARTITION_EDGE: { int numBuckets = edgeProp.getNumBuckets(); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); DataOutputBuffer dob = new DataOutputBuffer(); @@ -332,7 +332,7 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr w.setVertexManagerPlugin(desc); break; } - case SIMPLE_EDGE: { + case SORT_PARTITION_EDGE: { setupAutoReducerParallelism(edgeProp, w); break; } @@ -364,7 +364,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et1Conf.createDefaultBroadcastEdgeProperty(); - case CUSTOM_EDGE: + case CO_PARTITION_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et2Conf = UnorderedPartitionedKVEdgeConfig @@ -382,7 +382,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); - case CUSTOM_SIMPLE_EDGE: + case PARTITION_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et3Conf = UnorderedPartitionedKVEdgeConfig @@ -392,7 +392,7 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et3Conf.createDefaultEdgeProperty(); - case SIMPLE_EDGE: + case SORT_PARTITION_EDGE: default: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java index 3a6baca..54bd63c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java @@ -252,10 +252,10 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma ReduceSinkOperator.class); if (rootOp == null) { // likely we found a table scan operator - edgeType = EdgeType.CUSTOM_EDGE; + edgeType = EdgeType.CO_PARTITION_EDGE; } else { // we have found a reduce sink - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } } else { Operator rootOp = OperatorUtils.findSingleOperatorUpstreamJoinAccounted( @@ -263,16 +263,16 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma TableScanOperator.class); if (rootOp != null) { // likely we found a table scan operator - edgeType = EdgeType.CUSTOM_EDGE; + edgeType = EdgeType.CO_PARTITION_EDGE; } else { // we have found a reduce sink - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } } } else if (mapJoinOp.getConf().isDynamicPartitionHashJoin()) { - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + edgeType = EdgeType.PARTITION_EDGE; } - if (edgeType == EdgeType.CUSTOM_EDGE) { + if (edgeType == EdgeType.CO_PARTITION_EDGE) { // disable auto parallelism for bucket map joins parentRS.getConf().setReducerTraits(EnumSet.of(FIXED)); } @@ -284,7 +284,7 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma TezWork tezWork = context.currentTask.getWork(); LOG.debug("connecting "+parentWork.getName()+" with "+myWork.getName()); tezWork.connect(parentWork, myWork, edgeProp); - if (edgeType == EdgeType.CUSTOM_EDGE) { + if (edgeType == EdgeType.CO_PARTITION_EDGE) { tezWork.setVertexType(myWork, VertexType.INITIALIZED_EDGES); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index aee74ad..a640e81 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -500,15 +500,15 @@ public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork follo if (reduceWork.getReducer() instanceof MapJoinOperator) { MapJoinOperator joinOp = (MapJoinOperator) reduceWork.getReducer(); if (joinOp.getConf().isDynamicPartitionHashJoin()) { - return EdgeType.CUSTOM_SIMPLE_EDGE; + return EdgeType.PARTITION_EDGE; } } } if(!reduceSinkOperator.getConf().isOrdering()) { //if no sort keys are specified, use an edge that does not sort - return EdgeType.CUSTOM_SIMPLE_EDGE; + return EdgeType.PARTITION_EDGE; } - return EdgeType.SIMPLE_EDGE; + return EdgeType.SORT_PARTITION_EDGE; } public static void processDynamicMinMaxPushDownOperator( diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java index 97f3300..6e440ea 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java @@ -283,7 +283,7 @@ public Object process(Node nd, Stack stack, LOG.debug("connecting "+parentWork.getName()+" with "+work.getName()); TezEdgeProperty edgeProp = parentWorkMap.getValue(); tezWork.connect(parentWork, work, edgeProp); - if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) { + if (edgeProp.getEdgeType() == EdgeType.CO_PARTITION_EDGE) { tezWork.setVertexType(work, VertexType.INITIALIZED_EDGES); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java index a3aa12f..21ae21e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java @@ -23,11 +23,11 @@ public class TezEdgeProperty { public enum EdgeType { - SIMPLE_EDGE, + SORT_PARTITION_EDGE, BROADCAST_EDGE, CONTAINS, - CUSTOM_EDGE, - CUSTOM_SIMPLE_EDGE, + CO_PARTITION_EDGE, + PARTITION_EDGE, } private HiveConf hiveConf; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java index 5c012f3..c76c077 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java @@ -153,7 +153,7 @@ public Edge answer(InvocationOnMock invocation) throws Throwable { rws[0].setReducer(op); rws[1].setReducer(op); - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(mws[0], rws[0], edgeProp); work.connect(mws[1], rws[0], edgeProp); work.connect(rws[0], rws[1], edgeProp); diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java index c1e192c..ea72cb1 100644 --- ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java +++ ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java @@ -67,7 +67,7 @@ public void testConnect() throws Exception { BaseWork parent = nodes.get(0); BaseWork child = nodes.get(1); - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, child, edgeProp); Assert.assertEquals(work.getParents(child).size(), 1); @@ -84,7 +84,7 @@ public void testConnect() throws Exception { Assert.assertEquals(work.getChildren(w).size(), 0); } - Assert.assertEquals(work.getEdgeProperty(parent, child).getEdgeType(), EdgeType.SIMPLE_EDGE); + Assert.assertEquals(work.getEdgeProperty(parent, child).getEdgeType(), EdgeType.SORT_PARTITION_EDGE); } @Test @@ -117,7 +117,7 @@ public void testDisconnect() throws Exception { BaseWork parent = nodes.get(0); BaseWork children[] = {nodes.get(1), nodes.get(2)}; - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, children[0], edgeProp); work.connect(parent, children[1], edgeProp); @@ -136,7 +136,7 @@ public void testRemove() throws Exception { BaseWork parent = nodes.get(0); BaseWork children[] = {nodes.get(1), nodes.get(2)}; - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); work.connect(parent, children[0], edgeProp); work.connect(parent, children[1], edgeProp); @@ -151,7 +151,7 @@ public void testRemove() throws Exception { @Test public void testGetAllWork() throws Exception { - TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE); + TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SORT_PARTITION_EDGE); for (int i = 4; i > 0; --i) { work.connect(nodes.get(i), nodes.get(i-1), edgeProp); } diff --git ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out index d05bf64..cac6194 100644 --- ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out +++ ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acidtbldefault diff --git ql/src/test/results/clientpositive/llap/auto_join0.q.out ql/src/test/results/clientpositive/llap/auto_join0.q.out index cba6001..32d9461 100644 --- ql/src/test/results/clientpositive/llap/auto_join0.q.out +++ ql/src/test/results/clientpositive/llap/auto_join0.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( @@ -30,13 +30,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -54,25 +54,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -87,13 +69,13 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Reducer 6 + 1 Vertex 6 Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -109,7 +91,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -124,7 +106,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -142,7 +142,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( SELECT src1.key as k1, src1.value as v1, diff --git ql/src/test/results/clientpositive/llap/auto_join1.q.out ql/src/test/results/clientpositive/llap/auto_join1.q.out index 6a0a1d5..379c087 100644 --- ql/src/test/results/clientpositive/llap/auto_join1.q.out +++ ql/src/test/results/clientpositive/llap/auto_join1.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -47,7 +47,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -67,7 +67,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) diff --git ql/src/test/results/clientpositive/llap/auto_join21.q.out ql/src/test/results/clientpositive/llap/auto_join21.q.out index 97b36f7..1d5f09e 100644 --- ql/src/test/results/clientpositive/llap/auto_join21.q.out +++ ql/src/test/results/clientpositive/llap/auto_join21.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -30,36 +30,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -84,7 +55,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,6 +69,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_join29.q.out ql/src/test/results/clientpositive/llap/auto_join29.q.out index a3cc39d..44a75b9 100644 --- ql/src/test/results/clientpositive/llap/auto_join29.q.out +++ ql/src/test/results/clientpositive/llap/auto_join29.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -30,36 +30,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -84,7 +55,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,6 +69,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -628,11 +628,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -645,39 +645,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -702,7 +670,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 322 Data size: 171948 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -716,6 +684,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1246,11 +1246,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1266,36 +1266,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1320,7 +1291,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 974 Data size: 520116 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1334,6 +1305,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1876,11 +1876,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1896,33 +1896,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1947,7 +1921,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 987 Data size: 527058 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1961,6 +1935,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2503,11 +2503,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2523,39 +2523,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key > 10) and (key < 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2576,7 +2544,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2590,6 +2558,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2620,11 +2620,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -2640,36 +2640,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((key < 10) and (key > 10)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2694,7 +2665,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2708,6 +2679,35 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3238,11 +3238,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3261,8 +3261,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3274,7 +3274,21 @@ STAGE PLANS: Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -3290,7 +3304,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3306,20 +3320,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 320 Data size: 170880 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3350,11 +3350,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3370,7 +3370,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -3392,8 +3392,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3405,7 +3405,21 @@ STAGE PLANS: Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3421,20 +3435,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 960 Data size: 512640 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3487,11 +3487,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3509,8 +3509,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) @@ -3522,7 +3522,21 @@ STAGE PLANS: Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -3538,7 +3552,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src3 @@ -3554,20 +3568,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 314 Data size: 167676 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_join30.q.out ql/src/test/results/clientpositive/llap/auto_join30.q.out index a26db55..bbfd9d7 100644 --- ql/src/test/results/clientpositive/llap/auto_join30.q.out +++ ql/src/test/results/clientpositive/llap/auto_join30.q.out @@ -23,12 +23,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -46,7 +46,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -65,19 +77,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -92,7 +92,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -103,7 +103,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -169,12 +169,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -189,23 +189,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -220,7 +204,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -231,7 +215,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -246,7 +230,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -310,12 +310,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -330,7 +330,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -346,19 +358,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -373,7 +373,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -384,7 +384,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -456,13 +456,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -480,26 +480,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Map Operator Tree: TableScan alias: src @@ -518,19 +511,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -547,8 +528,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 - 2 Reducer 7 + 0 Vertex 2 + 2 Vertex 7 Statistics: Num rows: 2974 Data size: 529372 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -559,7 +540,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -574,7 +555,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -649,14 +649,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -671,39 +671,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -715,7 +683,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -737,7 +705,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -752,7 +720,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -765,7 +749,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -840,14 +840,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -862,39 +862,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -906,7 +874,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -928,7 +896,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -943,7 +911,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -956,7 +940,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1031,14 +1031,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1053,39 +1053,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1097,7 +1065,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1119,7 +1087,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1134,7 +1102,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1147,7 +1131,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1222,14 +1222,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1244,39 +1244,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1288,7 +1256,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1310,7 +1278,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1325,7 +1293,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1338,7 +1322,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/auto_join_filters.q.out ql/src/test/results/clientpositive/llap/auto_join_filters.q.out index d1d9408..f4c1e79 100644 --- ql/src/test/results/clientpositive/llap/auto_join_filters.q.out +++ ql/src/test/results/clientpositive/llap/auto_join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -34,7 +34,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -44,7 +44,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3080335 -Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -300,7 +300,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table sm POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_input2 -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -310,7 +310,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -320,7 +320,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -330,7 +330,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3080335 -Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[14][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out index 5984e8f..75254df 100644 --- ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out +++ ql/src/test/results/clientpositive/llap/auto_join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -34,7 +34,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out index a867bd2..a180b0d 100644 --- ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out @@ -57,10 +57,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -100,7 +100,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -166,11 +166,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -212,7 +212,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -231,7 +231,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -321,12 +321,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -370,7 +370,44 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: b @@ -414,44 +451,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -540,10 +540,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -583,7 +583,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -657,10 +657,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -700,7 +700,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -841,7 +841,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -927,10 +927,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -970,7 +970,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1034,11 +1034,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1057,26 +1057,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key + 1) is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (key + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1095,7 +1076,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1110,6 +1091,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (key + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1155,10 +1155,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1198,7 +1198,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1264,10 +1264,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1320,7 +1320,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1402,10 +1402,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1445,7 +1445,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1541,7 +1541,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1755,10 +1755,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1814,7 +1814,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out index 4b646dc..318542a 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out @@ -112,11 +112,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -194,7 +194,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -211,12 +211,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -337,7 +337,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -405,11 +405,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -426,12 +426,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 1 + Estimated key counts: Vertex 3 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -552,7 +552,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -630,37 +661,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -698,11 +698,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -719,12 +719,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 1 + Estimated key counts: Vertex 3 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -845,7 +845,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -923,37 +954,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out index ab94c37..dd6bf84 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_10.q.out @@ -71,12 +71,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -95,7 +97,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -108,7 +110,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -127,7 +144,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -140,7 +157,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -164,23 +181,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -240,11 +240,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -264,7 +264,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -283,7 +283,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -296,7 +296,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out index 7e3fe4c..0fca3d9 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out @@ -110,11 +110,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -191,7 +191,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -208,12 +208,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -332,7 +332,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -400,11 +400,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -481,7 +481,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -498,12 +498,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -622,7 +622,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -690,11 +690,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -767,7 +767,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -780,12 +780,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Vertex 1 => 1 keys: 0 key (type: string) 1 key (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -904,7 +904,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -972,11 +972,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1049,7 +1049,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1063,14 +1063,14 @@ STAGE PLANS: condition map: Inner Join 0 to 1 Inner Join 0 to 2 - Estimated key counts: Map 1 => 1, Map 4 => 116 + Estimated key counts: Vertex 1 => 1, Vertex 4 => 116 keys: 0 key (type: string) 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Position of Big Table: 1 Statistics: Num rows: 255 Data size: 27614 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1189,7 +1189,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Map 4 + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1311,37 +1342,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [c] /bucket_big/ds=2008-04-09 [c] - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out index 054cf1a..c2b611f 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out @@ -134,7 +134,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -148,11 +148,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE), Vertex 2 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -230,7 +230,7 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -308,7 +308,7 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [b] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -326,25 +326,25 @@ STAGE PLANS: condition map: Inner Join 0 to 1 Inner Join 1 to 2 - Estimated key counts: Map 1 => 1, Map 2 => 1 + Estimated key counts: Vertex 1 => 1, Vertex 2 => 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) 2 _col0 (type: string) input vertices: - 0 Map 1 - 1 Map 2 + 0 Vertex 1 + 1 Vertex 2 Position of Big Table: 2 Statistics: Num rows: 255 Data size: 27614 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 5 => 1 + Estimated key counts: Vertex 5 => 1 keys: 0 1 input vertices: - 1 Map 5 + 1 Vertex 5 Position of Big Table: 0 Statistics: Num rows: 255 Data size: 29909 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -465,7 +465,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [c] /bucket_big/ds=2008-04-09 [c] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 5 Map Operator Tree: TableScan alias: d @@ -535,37 +566,6 @@ STAGE PLANS: name: default.bucket_medium Truncated Path -> Alias: /bucket_medium/ds=2008-04-08 [d] - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -573,7 +573,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out index 9c37848..ba800b7 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out @@ -79,10 +79,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -102,7 +102,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -130,7 +130,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -295,10 +295,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -318,7 +318,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -346,7 +346,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -511,10 +511,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -534,7 +534,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int) @@ -562,7 +562,7 @@ STAGE PLANS: name: default.dest2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out index c618531..7dfb541 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out @@ -49,11 +49,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -69,7 +69,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -82,7 +82,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -98,21 +113,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -156,11 +156,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -176,7 +176,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -192,7 +192,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 207 Data size: 2080 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -205,7 +205,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out index 286939b..17c85ab 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out @@ -49,11 +49,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -69,7 +69,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -82,7 +82,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -98,21 +113,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -135,11 +135,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -155,7 +155,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -171,7 +171,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -184,7 +184,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out index 42e4cb7..ac23dd5 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out @@ -94,11 +94,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -115,12 +115,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -241,7 +241,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -319,37 +350,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -408,12 +408,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -534,7 +534,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -612,37 +643,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out index 64c10ce..6ae25ab 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out @@ -94,11 +94,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -226,7 +226,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -243,12 +243,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -319,7 +319,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -408,12 +408,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -484,7 +484,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -612,37 +643,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -680,11 +680,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -701,12 +701,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 63 Data size: 6903 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -777,7 +777,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -905,37 +936,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out index 96854a9..e62cb2e 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out @@ -110,11 +110,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -242,7 +242,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -259,12 +259,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 4 + Estimated key counts: Vertex 1 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -335,7 +335,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -403,11 +403,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -424,12 +424,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -500,7 +500,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -628,37 +659,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -696,11 +696,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -717,12 +717,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3262 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -793,7 +793,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -921,37 +952,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out index b77a879..8ecb45f 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out @@ -75,10 +75,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -230,7 +230,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -292,10 +292,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -447,7 +447,7 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [a] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -509,11 +509,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -530,12 +530,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -605,7 +605,38 @@ STAGE PLANS: name: default.bucket_big Truncated Path -> Alias: /bucket_big [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -682,37 +713,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out index 9e60536..35245e2 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_6.q.out @@ -83,11 +83,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -124,26 +124,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -162,7 +143,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -177,6 +158,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -210,11 +210,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -251,26 +251,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: d - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -289,7 +270,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -304,6 +285,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -337,11 +337,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -378,26 +378,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: h - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -416,7 +397,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -431,6 +412,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: h + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -464,11 +464,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -505,26 +505,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -543,7 +524,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -558,6 +539,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -591,10 +591,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -647,7 +647,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -695,11 +695,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -736,26 +736,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -774,7 +755,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -789,6 +770,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -822,11 +822,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -863,26 +863,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -901,7 +882,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -916,6 +897,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -949,11 +949,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -990,26 +990,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1028,7 +1009,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1043,6 +1024,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1076,10 +1076,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1132,7 +1132,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1180,11 +1180,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1221,26 +1221,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1259,7 +1240,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1274,6 +1255,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out index 13d3acc..d861a94 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out @@ -127,11 +127,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -259,7 +259,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -276,12 +276,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 4 + Estimated key counts: Vertex 1 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -402,7 +402,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -472,11 +472,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -493,12 +493,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -619,7 +619,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -747,37 +778,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -817,11 +817,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -838,12 +838,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 4 + Estimated key counts: Vertex 3 => 4 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 59 Data size: 6525 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -964,7 +964,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1092,37 +1123,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out index 7fc90fd..9a96912 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out @@ -127,11 +127,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -259,7 +259,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [a] /bucket_small/ds=2008-04-09 [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -276,12 +276,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -402,7 +402,7 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [b] /bucket_big/ds=2008-04-09 [b] - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -472,11 +472,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -493,12 +493,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -619,7 +619,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -747,37 +778,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -817,11 +817,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -838,12 +838,12 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 2 + Estimated key counts: Vertex 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 127 Data size: 13807 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -964,7 +964,38 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_big/ds=2008-04-08 [a] /bucket_big/ds=2008-04-09 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1092,37 +1123,6 @@ STAGE PLANS: Truncated Path -> Alias: /bucket_small/ds=2008-04-08 [b] /bucket_small/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out index c08a534..3768165 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out @@ -57,11 +57,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -80,7 +80,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -93,7 +93,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -112,21 +127,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -172,11 +172,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -196,7 +196,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -212,7 +212,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -231,22 +247,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -307,12 +307,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -332,7 +332,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) @@ -346,26 +346,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -384,7 +365,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -399,6 +380,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -474,13 +474,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -500,7 +500,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -516,7 +516,37 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -535,7 +565,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -555,7 +585,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -571,7 +601,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: b @@ -590,51 +635,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Reducer 5 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -709,11 +709,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -732,7 +732,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -745,7 +745,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -764,21 +779,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -838,11 +838,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -861,7 +861,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -874,7 +874,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -893,21 +908,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -991,11 +991,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1014,7 +1014,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1027,7 +1027,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1046,21 +1061,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1132,11 +1132,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1155,7 +1155,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1168,7 +1168,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1187,21 +1202,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1251,11 +1251,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1274,7 +1274,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1287,7 +1287,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1306,21 +1321,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1366,11 +1366,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1389,7 +1389,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1402,7 +1402,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1421,21 +1436,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1479,11 +1479,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1502,7 +1502,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1515,7 +1515,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1534,21 +1549,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1602,11 +1602,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1627,8 +1627,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1641,7 +1641,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -1660,7 +1675,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -1679,21 +1694,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1761,11 +1761,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1784,7 +1784,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1797,7 +1797,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1816,21 +1831,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1888,11 +1888,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1911,7 +1911,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1924,7 +1924,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1943,21 +1958,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2003,11 +2003,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2027,7 +2027,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2043,7 +2043,23 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2062,22 +2078,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2138,12 +2138,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2163,7 +2163,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) @@ -2177,26 +2177,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2215,7 +2196,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2230,6 +2211,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2305,13 +2305,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2331,7 +2331,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2347,7 +2347,37 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2366,7 +2396,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -2386,72 +2416,23 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Reducer 5 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Group By Operator + aggregations: count() + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2466,6 +2447,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2540,11 +2540,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2563,7 +2563,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2576,7 +2576,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -2595,21 +2610,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2669,11 +2669,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2692,7 +2692,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2705,7 +2705,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2724,21 +2739,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2822,11 +2822,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2845,7 +2845,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2858,7 +2858,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -2877,21 +2892,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2963,11 +2963,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2986,7 +2986,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2999,7 +2999,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3018,21 +3033,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3078,11 +3078,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3101,7 +3101,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3114,7 +3114,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3133,21 +3148,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3191,11 +3191,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3214,7 +3214,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3227,7 +3227,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3246,21 +3261,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3314,11 +3314,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3339,8 +3339,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3353,7 +3353,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -3372,7 +3387,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: a @@ -3391,21 +3406,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -3473,11 +3473,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -3496,7 +3496,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3509,7 +3509,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -3528,21 +3543,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucket2.q.out ql/src/test/results/clientpositive/llap/bucket2.q.out index be886b0..dc5db82 100644 --- ql/src/test/results/clientpositive/llap/bucket2.q.out +++ ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket3.q.out ql/src/test/results/clientpositive/llap/bucket3.q.out index 8e6d85c..3279500 100644 --- ql/src/test/results/clientpositive/llap/bucket3.q.out +++ ql/src/test/results/clientpositive/llap/bucket3.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket4.q.out ql/src/test/results/clientpositive/llap/bucket4.q.out index b4f7ca1..88445a2 100644 --- ql/src/test/results/clientpositive/llap/bucket4.q.out +++ ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -100,7 +100,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket5.q.out ql/src/test/results/clientpositive/llap/bucket5.q.out index ef5067b..f5647e6 100644 --- ql/src/test/results/clientpositive/llap/bucket5.q.out +++ ql/src/test/results/clientpositive/llap/bucket5.q.out @@ -42,11 +42,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -131,7 +131,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -173,7 +173,7 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket6.q.out ql/src/test/results/clientpositive/llap/bucket6.q.out index 20895f8..d25a965 100644 --- ql/src/test/results/clientpositive/llap/bucket6.q.out +++ ql/src/test/results/clientpositive/llap/bucket6.q.out @@ -23,10 +23,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -43,7 +43,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/bucket_groupby.q.out ql/src/test/results/clientpositive/llap/bucket_groupby.q.out index 2c3cf2f..01618ef 100644 --- ql/src/test/results/clientpositive/llap/bucket_groupby.q.out +++ ql/src/test/results/clientpositive/llap/bucket_groupby.q.out @@ -55,11 +55,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -83,7 +83,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -98,7 +98,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -183,11 +183,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -211,7 +211,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -226,7 +226,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -285,10 +285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -312,7 +312,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -366,10 +366,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -393,7 +393,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -447,11 +447,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -475,7 +475,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -490,7 +490,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -549,11 +549,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -577,7 +577,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -592,7 +592,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -651,10 +651,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -677,7 +677,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1036,10 +1036,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1062,7 +1062,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1150,11 +1150,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1179,7 +1179,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1194,7 +1194,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1253,11 +1253,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1281,7 +1281,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1296,7 +1296,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1456,11 +1456,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1485,7 +1485,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1500,7 +1500,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1559,11 +1559,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: clustergroupby @@ -1587,7 +1587,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1606,7 +1606,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 4656 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/bucket_many.q.out ql/src/test/results/clientpositive/llap/bucket_many.q.out index 72b0ec4..28b9424 100644 --- ql/src/test/results/clientpositive/llap/bucket_many.q.out +++ ql/src/test/results/clientpositive/llap/bucket_many.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 964d058..5896005 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -125,10 +125,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -148,7 +148,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -168,7 +168,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -209,11 +209,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -233,26 +233,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -271,7 +252,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -282,7 +263,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -297,6 +278,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -346,11 +346,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE), Map 4 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 4 (CO_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -369,7 +369,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -389,7 +389,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -398,7 +398,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -411,7 +411,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d @@ -430,21 +445,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -498,11 +498,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 1 (CUSTOM_EDGE), Map 2 (CUSTOM_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (CO_PARTITION_EDGE), Vertex 2 (CO_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d @@ -521,7 +521,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: a @@ -540,7 +540,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -560,7 +560,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 0 Map 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -569,7 +569,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -582,7 +582,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -646,11 +646,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -676,7 +676,26 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -696,7 +715,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) @@ -711,25 +730,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double) Stage: Stage-0 Fetch Operator @@ -756,11 +756,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -780,7 +780,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), substr(_col2, 5) (type: string) @@ -800,7 +800,41 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: tab @@ -820,7 +854,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: b @@ -840,40 +874,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 5121 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 4 - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 302 Data size: 5633 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -900,11 +900,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -924,7 +924,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), substr(_col1, 5) (type: string) @@ -944,7 +944,41 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: y @@ -963,7 +997,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: b @@ -983,40 +1017,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 133 Data size: 2476 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 4 - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: int), _col0 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1041,10 +1041,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1066,8 +1066,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 2 - 2 Map 3 + 1 Vertex 2 + 2 Vertex 3 Statistics: Num rows: 1100 Data size: 20486 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1082,7 +1082,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1102,7 +1102,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -1145,10 +1145,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1168,7 +1168,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1178,7 +1178,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1193,7 +1193,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: y @@ -1212,7 +1212,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: c @@ -1258,11 +1258,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -1288,7 +1288,22 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1308,7 +1323,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) @@ -1323,21 +1338,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) Stage: Stage-0 Fetch Operator @@ -1364,11 +1364,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Reducer 2 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_mapjoin @@ -1388,7 +1388,22 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1408,7 +1423,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: double), _col3 (type: string) @@ -1423,21 +1438,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: double) Stage: Stage-0 Fetch Operator @@ -1462,10 +1462,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1485,7 +1485,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1505,7 +1505,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1562,10 +1562,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1585,7 +1585,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1605,7 +1605,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -1640,10 +1640,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1663,7 +1663,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1673,7 +1673,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -1688,7 +1688,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: c @@ -1707,7 +1707,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1751,10 +1751,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1786,10 +1786,10 @@ STAGE PLANS: Target Input: b Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Target Vertex: Map 2 + Target Vertex: Vertex 2 Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1809,7 +1809,7 @@ STAGE PLANS: 1 _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 60500 Data size: 12826000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: int), _col1 (type: string), _col4 (type: string) @@ -1862,11 +1862,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1885,14 +1885,29 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count() + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1911,21 +1926,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 19 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count() - mode: complete - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1981,11 +1981,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2004,14 +2004,29 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count() + mode: complete + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2030,21 +2045,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 19 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count() - mode: complete - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out index f6500eb..ded7ccd 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez2.q.out @@ -121,10 +121,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -144,7 +144,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -154,7 +154,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 605 Data size: 11267 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -169,7 +169,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: c @@ -188,7 +188,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: b @@ -254,10 +254,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -277,7 +277,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -297,7 +297,7 @@ STAGE PLANS: 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) @@ -334,10 +334,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -357,7 +357,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 60 Data size: 1126 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -368,7 +368,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -409,10 +409,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (CUSTOM_EDGE) + Vertex 1 <- Vertex 2 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -432,7 +432,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 182 Data size: 3400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -443,7 +443,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -484,10 +484,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (CUSTOM_EDGE) + Vertex 2 <- Vertex 1 (CO_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab_part @@ -506,7 +506,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 1024 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tab_part @@ -526,7 +526,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 182 Data size: 3400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -557,10 +557,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab @@ -580,26 +580,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -615,7 +596,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -624,6 +605,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -644,10 +644,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tab @@ -667,27 +667,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -703,7 +683,7 @@ STAGE PLANS: 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string) @@ -716,6 +696,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out index e6cfb40..07df57d 100644 --- ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out +++ ql/src/test/results/clientpositive/llap/bucketizedhiveinputformat.q.out @@ -22,7 +22,7 @@ POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@T2 -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM ( SELECT tmp1.name as name FROM ( SELECT name, 'MMM' AS n FROM T1) tmp1 diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out index 93efa3c..b0e12d7 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out @@ -41,10 +41,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -65,28 +65,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: key (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: value (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -125,6 +104,27 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: key (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: value (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -165,10 +165,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -189,28 +189,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: key (type: int) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: value (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -249,6 +228,27 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (key is not null and (ds = '2008-04-08')) (type: boolean) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: key (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: value (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -383,10 +383,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -458,7 +458,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -532,55 +581,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -741,10 +741,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -816,7 +816,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -890,55 +939,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out index cea632b..74adfbd 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out @@ -111,10 +111,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -188,7 +188,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 0 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -262,55 +311,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -475,10 +475,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -552,7 +552,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 0 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -626,55 +675,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 0 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -856,10 +856,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -933,7 +933,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1056,55 +1105,6 @@ STAGE PLANS: Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [b] /srcbucket_mapjoin_part_2/ds=2008-04-09 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 63 Data size: 7246 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out index 4aa0e0e..fd6952a 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -212,7 +212,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -286,55 +335,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -499,10 +499,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -576,7 +576,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08 [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 564 + rawDataSize 10503 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 11067 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -650,55 +699,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col7 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 564 - rawDataSize 10503 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 11067 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out index 21ce1f8..f27803d 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -210,7 +210,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -282,55 +331,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -483,10 +483,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -558,7 +558,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 0 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -630,55 +679,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 0 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out index 3a2cbb2..8168a02 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out @@ -73,11 +73,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -151,7 +151,66 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_1 Truncated Path -> Alias: /srcbucket_mapjoin_part_1/ds=2008-04-08/hr=0 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col8 + Position of Big Table: 0 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: aa + sort order: ++ + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 1 + TopN Hash Memory Usage: 0.1 + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -226,65 +285,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part_2 Truncated Path -> Alias: /srcbucket_mapjoin_part_2/ds=2008-04-08/hr=0 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col8 - Position of Big Table: 0 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: aa - sort order: ++ - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 1 - TopN Hash Memory Usage: 0.1 - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 755 Data size: 9070 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/bucketpruning1.q.out ql/src/test/results/clientpositive/llap/bucketpruning1.q.out index 922f23a..040003e 100644 --- ql/src/test/results/clientpositive/llap/bucketpruning1.q.out +++ ql/src/test/results/clientpositive/llap/bucketpruning1.q.out @@ -29,7 +29,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -91,7 +91,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -153,7 +153,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -215,7 +215,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -277,7 +277,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -339,7 +339,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -401,7 +401,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -463,7 +463,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -525,7 +525,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -587,7 +587,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -649,7 +649,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -711,7 +711,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -773,7 +773,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -834,7 +834,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -895,7 +895,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -957,7 +957,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1018,7 +1018,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1079,7 +1079,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1140,7 +1140,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1201,7 +1201,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1262,7 +1262,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1323,7 +1323,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1384,7 +1384,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1444,7 +1444,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1505,7 +1505,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_pruned @@ -1566,7 +1566,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_unpruned @@ -1627,7 +1627,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcbucket_unpruned diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out index e4fed11..3690558 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out @@ -99,11 +99,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -123,7 +123,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -143,7 +143,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -157,7 +157,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -278,11 +278,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -302,7 +302,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -322,7 +322,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -336,7 +336,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -481,11 +481,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -505,7 +505,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -525,7 +525,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 184 Data size: 3097 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -539,7 +539,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -690,11 +690,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -714,7 +714,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -734,7 +734,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -748,7 +748,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -881,11 +881,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -905,7 +905,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -925,7 +925,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -939,7 +939,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1072,11 +1072,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -1096,7 +1096,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -1116,7 +1116,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string) @@ -1130,7 +1130,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out index 67e925a..0bb8b77 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out @@ -77,11 +77,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -101,7 +101,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -121,7 +121,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -263,11 +263,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -287,7 +287,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -307,7 +307,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -321,7 +321,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -449,11 +449,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -473,7 +473,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -493,7 +493,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col6) (type: string) @@ -507,7 +507,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -570,11 +570,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -594,7 +594,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -614,7 +614,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col6) (type: string) @@ -628,7 +628,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -697,11 +697,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -721,7 +721,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -741,7 +741,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -755,7 +755,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -901,11 +901,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -925,7 +925,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -945,7 +945,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -959,7 +959,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1115,11 +1115,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1139,7 +1139,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1159,7 +1159,7 @@ STAGE PLANS: 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col6 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1816 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col6) (type: string) @@ -1173,7 +1173,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out index 8691d0d..f7f6e90 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out @@ -77,11 +77,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -101,7 +101,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -121,7 +121,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col4) (type: string) @@ -135,7 +135,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -260,11 +260,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -284,7 +284,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -304,7 +304,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 92 Data size: 1548 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -318,7 +318,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -449,11 +449,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_table1 @@ -473,7 +473,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_table2 @@ -493,7 +493,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 30 Data size: 515 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), concat(_col1, _col3) (type: string) @@ -507,7 +507,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/cluster.q.out ql/src/test/results/clientpositive/llap/cluster.q.out index 8c89ee3..d0ee03a 100644 --- ql/src/test/results/clientpositive/llap/cluster.q.out +++ ql/src/test/results/clientpositive/llap/cluster.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -36,7 +36,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -149,10 +149,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -172,7 +172,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -217,10 +217,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -240,7 +240,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -285,10 +285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -308,7 +308,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -353,10 +353,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -376,7 +376,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,10 +421,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -444,7 +444,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -489,10 +489,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -512,7 +512,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -557,11 +557,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -577,22 +577,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -613,7 +598,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 4 Data size: 1060 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -627,6 +612,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -658,11 +658,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -678,23 +678,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -715,7 +699,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 4 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -729,6 +713,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -760,11 +760,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -780,23 +780,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -817,7 +801,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -831,6 +815,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -862,11 +862,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -882,22 +882,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = 20) (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -918,7 +903,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 1060 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -932,6 +917,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -975,12 +975,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1000,7 +1002,21 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -1020,22 +1036,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 332 Data size: 59096 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out index 80ccddd..47df121 100644 --- ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out +++ ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out @@ -123,10 +123,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -308,10 +308,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -331,7 +331,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -398,7 +398,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -508,10 +508,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendar @@ -531,7 +531,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -694,10 +694,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendarp @@ -717,7 +717,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -870,10 +870,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: calendarp @@ -893,7 +893,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/column_access_stats.q.out ql/src/test/results/clientpositive/llap/column_access_stats.q.out index c56c818..7b9bfd9 100644 --- ql/src/test/results/clientpositive/llap/column_access_stats.q.out +++ ql/src/test/results/clientpositive/llap/column_access_stats.q.out @@ -177,11 +177,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -199,7 +201,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1 @@ -217,8 +219,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -265,11 +265,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -287,7 +289,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1 @@ -305,8 +307,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -380,10 +380,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -402,7 +402,25 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -421,24 +439,6 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -511,10 +511,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -534,7 +534,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -554,24 +572,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -613,10 +613,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -635,7 +635,25 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -654,24 +672,6 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -726,11 +726,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -749,7 +749,41 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: t2 @@ -768,7 +802,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: t3 @@ -788,40 +822,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/constprog_dpp.q.out ql/src/test/results/clientpositive/llap/constprog_dpp.q.out index 8f51db3..9b34fbf 100644 --- ql/src/test/results/clientpositive/llap/constprog_dpp.q.out +++ ql/src/test/results/clientpositive/llap/constprog_dpp.q.out @@ -41,47 +41,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 3 <- Union 4 (CONTAINS) -Map 6 <- Union 4 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 5 <- Union 4 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 4 (CONTAINS) +Vertex 5 <- Union 4 (PARTITION_EDGE) +Vertex 6 <- Union 4 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_17] - Merge Join Operator [MERGEJOIN_21] (rows=1 width=0) - Conds:RS_13._col0=RS_14._col0(Left Outer),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=1 width=0) - Output:["_col0"] - TableScan [TS_0] (rows=1 width=0) - default@tb1,a,Tbl:PARTIAL,Col:NONE,Output:["id"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Limit [LIM_11] (rows=1 width=0) - Number of rows:1 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - <-Union 4 [CUSTOM_SIMPLE_EDGE] - <-Map 3 [CONTAINS] llap - Reduce Output Operator [RS_9] - Limit [LIM_8] (rows=1 width=0) - Number of rows:1 - Select Operator [SEL_3] (rows=1 width=0) - Output:["_col0"] - TableScan [TS_2] (rows=1 width=0) - Output:["id"] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_9] - Limit [LIM_8] (rows=1 width=0) - Number of rows:1 - Select Operator [SEL_5] (rows=1 width=0) - Output:["_col0"] - TableScan [TS_4] (rows=1 width=0) + Vertex 2 diff --git ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out index 98d2328..763f217 100644 --- ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/llap/constprog_semijoin.q.out @@ -54,38 +54,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=5 width=22) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8._col3=RS_9._col0(Left Semi),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col2","_col3"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((val = 't1val01') and dimid is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=5 width=3) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=5 width=3) - Output:["_col0"] - Filter Operator [FIL_16] (rows=5 width=3) - predicate:id is not null - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01' PREHOOK: type: QUERY @@ -107,53 +82,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_18] - Select Operator [SEL_17] (rows=5 width=24) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_28] (rows=5 width=24) - Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col0","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=3 width=10) - predicate:id is not null - TableScan [TS_6] (rows=3 width=10) - default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_27] (rows=5 width=22) - Conds:RS_11._col2=RS_12._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col2"] - Filter Operator [FIL_24] (rows=5 width=20) - predicate:((val = 't1val01') and id is not null and dimid is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","dimid"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=5 width=3) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=5 width=3) - Output:["_col0"] - Filter Operator [FIL_25] (rows=5 width=3) - predicate:id is not null - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 3 PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id PREHOOK: type: QUERY @@ -177,53 +113,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_18] - Select Operator [SEL_17] (rows=5 width=24) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_28] (rows=5 width=24) - Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col0","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=3 width=10) - predicate:id is not null - TableScan [TS_6] (rows=3 width=10) - default@table2,table2,Tbl:COMPLETE,Col:NONE,Output:["id","val2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_27] (rows=5 width=22) - Conds:RS_11._col2=RS_12._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col2"] - Filter Operator [FIL_24] (rows=5 width=20) - predicate:((val = 't1val01') and dimid is not null and id is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","dimid"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=5 width=3) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=5 width=3) - Output:["_col0"] - Filter Operator [FIL_25] (rows=5 width=3) - predicate:id is not null - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 3 PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id PREHOOK: type: QUERY @@ -247,36 +144,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=1 width=20) - predicate:false - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100 PREHOOK: type: QUERY @@ -295,36 +169,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((dimid = 100) and (dimid = 100) is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200) PREHOOK: type: QUERY @@ -345,36 +196,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=1 width=20) - predicate:false - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200 PREHOOK: type: QUERY @@ -393,36 +221,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((dimid = 100) and (dimid = 100) is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100 PREHOOK: type: QUERY @@ -443,36 +248,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:100, true - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:((dimid = 100) and (dimid = 100) is not null) - TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=2 width=3) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=2 width=3) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=2 width=3) - predicate:((id = 100) and (id = 100) is not null) - TableScan [TS_3] (rows=5 width=3) - default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out index 86d94b4..ebd7018 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out @@ -19,12 +19,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -43,26 +43,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +66,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -107,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -122,6 +103,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -167,12 +167,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -191,26 +191,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -233,7 +214,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -255,7 +236,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -270,6 +251,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -315,12 +315,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -339,7 +339,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: y @@ -359,7 +359,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -375,7 +375,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -397,7 +397,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -457,12 +457,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -481,31 +481,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -528,7 +504,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -550,7 +526,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -565,6 +541,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -610,12 +610,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -634,31 +634,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -681,7 +657,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -703,7 +679,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -718,6 +694,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -763,12 +763,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -784,23 +784,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -823,7 +807,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -845,7 +829,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -860,6 +844,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -905,12 +905,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -926,23 +926,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -965,7 +949,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -987,7 +971,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1002,6 +986,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1047,12 +1047,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1068,23 +1068,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1111,7 +1095,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1133,7 +1117,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1148,6 +1132,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1193,12 +1193,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1214,23 +1214,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1257,7 +1241,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1279,7 +1263,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1294,6 +1278,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1337,11 +1337,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1357,23 +1357,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1400,7 +1384,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 12 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1416,6 +1400,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1472,11 +1472,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1492,23 +1492,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1535,7 +1519,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 12 Data size: 2220 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1551,6 +1535,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1609,12 +1609,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1630,23 +1630,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1673,7 +1657,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1695,7 +1679,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1710,6 +1694,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1755,12 +1755,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1776,23 +1776,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1819,7 +1803,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1841,7 +1825,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1856,6 +1840,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1901,12 +1901,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1922,23 +1922,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1961,7 +1945,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1983,7 +1967,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1998,6 +1982,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2043,12 +2043,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2064,23 +2064,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2103,7 +2087,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2125,7 +2109,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2140,6 +2124,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2185,12 +2185,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2206,23 +2206,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2245,7 +2229,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2267,7 +2251,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2282,6 +2266,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2327,12 +2327,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2348,23 +2348,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2387,7 +2371,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2409,7 +2393,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2424,6 +2408,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2469,12 +2469,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2494,26 +2494,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2536,7 +2517,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5490 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2558,7 +2539,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2573,6 +2554,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2618,12 +2618,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2643,26 +2643,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2685,7 +2666,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5490 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2707,7 +2688,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2722,6 +2703,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2767,12 +2767,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2791,26 +2791,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2833,7 +2814,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2855,7 +2836,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2870,6 +2851,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2915,12 +2915,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2939,26 +2939,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2981,7 +2962,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3003,7 +2984,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3018,6 +2999,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out index b628cb1..41afbe8 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out @@ -21,13 +21,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -49,29 +49,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -86,7 +64,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -110,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -125,7 +103,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -189,13 +189,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -217,29 +217,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -254,7 +232,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -278,7 +256,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -293,7 +271,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -357,13 +357,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -386,26 +386,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -420,7 +401,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -444,7 +425,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -459,7 +440,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -523,13 +523,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -552,26 +552,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -586,7 +567,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -610,7 +591,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -625,7 +606,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -689,13 +689,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -714,30 +714,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -752,7 +729,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -776,7 +753,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -791,7 +768,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -855,13 +855,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -880,30 +880,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -918,7 +895,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -942,7 +919,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -957,7 +934,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1021,13 +1021,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1050,30 +1050,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1088,7 +1065,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1112,7 +1089,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1127,7 +1104,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1191,13 +1191,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1220,30 +1220,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1258,7 +1235,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1282,7 +1259,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1297,7 +1274,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1363,14 +1363,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1391,28 +1391,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1425,7 +1404,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1448,7 +1427,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1470,7 +1449,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1485,7 +1464,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1551,14 +1551,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1579,28 +1579,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1613,7 +1592,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1636,7 +1615,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1330 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1658,7 +1637,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1673,7 +1652,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1737,13 +1737,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1762,49 +1762,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1825,7 +1783,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1849,7 +1807,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1864,7 +1822,49 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1928,13 +1928,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1953,49 +1953,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2016,7 +1974,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2040,7 +1998,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2055,7 +2013,49 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out index d3cfce8..0fbbdbe 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out @@ -21,14 +21,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -47,65 +47,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -126,7 +68,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -150,7 +92,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -165,7 +107,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -192,7 +173,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -207,6 +188,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -256,53 +256,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -321,26 +282,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -361,7 +303,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -385,7 +327,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -400,7 +342,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -427,7 +408,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -442,6 +423,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -491,13 +491,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 5 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -517,7 +517,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -531,7 +531,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 10980 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0) (type: int), hash(_col3) (type: int), hash(_col1) (type: int) @@ -548,7 +548,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -568,7 +583,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -588,7 +603,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -608,7 +623,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: x @@ -627,36 +657,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -691,68 +691,29 @@ FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value ON b.key = d.key) tmp PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)), SUM(HASH(tmp.value)) -FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value - FROM (SELECT x.key, x.value FROM src1 x JOIN src y ON (x.key = y.key)) b - JOIN (SELECT x.key, count(1) AS cnt FROM src1 x JOIN src y ON (x.key = y.key) group by x.key) d - ON b.key = d.key) tmp -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 +SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)), SUM(HASH(tmp.value)) +FROM (SELECT d.key AS key, d.cnt AS cnt, b.value as value + FROM (SELECT x.key, x.value FROM src1 x JOIN src y ON (x.key = y.key)) b + JOIN (SELECT x.key, count(1) AS cnt FROM src1 x JOIN src y ON (x.key = y.key) group by x.key) d + ON b.key = d.key) tmp +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: y @@ -771,26 +732,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -811,7 +753,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -835,7 +777,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -850,7 +792,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -877,7 +858,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -892,6 +873,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -941,53 +941,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1006,26 +967,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1046,7 +988,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1070,7 +1012,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1085,7 +1027,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1112,7 +1093,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1127,6 +1108,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1176,13 +1176,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 5 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -1202,7 +1202,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -1216,7 +1216,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 10980 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col2) (type: int), hash(_col3) (type: int), hash(_col1) (type: int) @@ -1233,7 +1233,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -1253,7 +1268,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -1273,7 +1288,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -1293,7 +1308,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: x @@ -1312,36 +1342,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out index 2034f8e..e745752a 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out @@ -67,12 +67,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -91,45 +91,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -158,7 +120,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -180,7 +142,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -195,6 +157,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -242,12 +242,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -266,45 +266,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -333,7 +295,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -355,7 +317,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -370,6 +332,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -417,12 +417,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -444,8 +444,8 @@ STAGE PLANS: 2 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 4 - 2 Map 5 + 1 Vertex 4 + 2 Vertex 5 Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int) @@ -465,7 +465,44 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hash(_col0) (type: int), hash(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col0), sum(_col1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y @@ -484,7 +521,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z @@ -503,43 +540,6 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hash(_col0) (type: int), hash(_col1) (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(_col0), sum(_col1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -587,12 +587,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -608,39 +608,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -665,7 +633,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -687,7 +655,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -702,6 +670,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -749,12 +749,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -770,39 +770,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -827,7 +795,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -849,7 +817,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -864,6 +832,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -911,12 +911,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -932,39 +932,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -993,7 +961,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1015,7 +983,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1030,6 +998,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1077,12 +1077,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1098,39 +1098,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1159,7 +1127,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1181,21 +1149,53 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1243,12 +1243,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1264,39 +1264,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1325,7 +1293,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1347,7 +1315,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1362,6 +1330,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1409,12 +1409,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1430,39 +1430,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1491,7 +1459,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1513,7 +1481,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1528,6 +1496,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1575,12 +1575,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1596,39 +1596,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1657,7 +1625,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1679,7 +1647,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1694,6 +1662,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1741,12 +1741,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1762,39 +1762,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1823,7 +1791,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 66 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1845,7 +1813,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1860,6 +1828,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out index 4fec286..45e7e50 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out @@ -23,14 +23,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -49,64 +49,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -129,7 +72,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -144,7 +87,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -162,7 +105,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -185,7 +166,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -200,6 +181,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -267,14 +267,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -293,64 +293,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -373,7 +316,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -388,7 +331,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -406,7 +349,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -429,7 +410,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -444,6 +425,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -511,13 +511,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 2 (BROADCAST_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -537,7 +537,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 44 Data size: 3784 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -553,7 +553,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 Map Operator Tree: TableScan alias: y @@ -572,7 +587,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -592,7 +607,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -608,7 +623,33 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 0 Vertex 2 + Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -627,47 +668,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - input vertices: - 0 Reducer 2 - Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 14 Data size: 2646 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -733,11 +733,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -756,7 +756,25 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -782,25 +800,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -878,11 +878,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -901,7 +901,25 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: x @@ -927,25 +945,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1023,12 +1023,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -1047,29 +1047,28 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: y + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -1085,25 +1084,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1126,7 +1107,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1141,6 +1122,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1204,12 +1204,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: xx @@ -1228,29 +1228,28 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: y + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -1266,25 +1265,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 4525 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1307,7 +1288,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1322,6 +1303,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1385,12 +1385,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1409,45 +1409,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: yy - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1470,7 +1432,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1485,7 +1447,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1503,6 +1465,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: yy + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1588,12 +1588,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1612,45 +1612,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: yy - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1673,7 +1635,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1688,7 +1650,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1706,6 +1668,44 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: yy + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1793,12 +1793,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: zz @@ -1817,7 +1817,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: xx @@ -1836,7 +1860,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -1855,50 +1879,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1921,7 +1902,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1936,6 +1917,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 7 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2025,12 +2025,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: zz @@ -2049,7 +2049,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: xx @@ -2068,7 +2092,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2087,50 +2111,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10860 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2153,7 +2134,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2168,6 +2149,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 7 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2257,12 +2257,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2281,64 +2281,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: zz - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: xx - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2361,7 +2304,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2376,7 +2319,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2400,6 +2343,63 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: zz + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: xx + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2461,81 +2461,43 @@ POSTHOOK: Input: default@src1 406 406 16 406 406 16 406 406 16 -66 66 1 -98 98 4 -98 98 4 -PREHOOK: query: EXPLAIN -SELECT xx.key, yy.key, yy.cnt -FROM src1 xx -JOIN -(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy -ON xx.key=yy.key JOIN src zz -ON yy.key=zz.key -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT xx.key, yy.key, yy.cnt -FROM src1 xx -JOIN -(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy -ON xx.key=yy.key JOIN src zz -ON yy.key=zz.key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 +66 66 1 +98 98 4 +98 98 4 +PREHOOK: query: EXPLAIN +SELECT xx.key, yy.key, yy.cnt +FROM src1 xx +JOIN +(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy +ON xx.key=yy.key JOIN src zz +ON yy.key=zz.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT xx.key, yy.key, yy.cnt +FROM src1 xx +JOIN +(SELECT x.key as key, count(1) as cnt FROM src x JOIN src y ON (x.key = y.key) group by x.key) yy +ON xx.key=yy.key JOIN src zz +ON yy.key=zz.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: zz + alias: x Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: key is not null (type: boolean) @@ -2551,26 +2513,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: xx - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2593,7 +2536,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2608,7 +2551,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2632,6 +2575,63 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: zz + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: xx + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2723,14 +2723,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2752,49 +2752,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2809,7 +2767,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2832,7 +2790,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2847,7 +2805,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2869,7 +2827,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2884,6 +2864,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 8 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2977,14 +2977,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3006,49 +3006,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3063,7 +3021,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3086,7 +3044,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3101,7 +3059,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 103 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3123,7 +3081,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3138,6 +3118,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 8 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3229,15 +3229,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3256,7 +3256,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -3275,46 +3275,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3337,7 +3298,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3352,7 +3313,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3367,7 +3328,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string), _col4 (type: bigint) sort order: +++++ Statistics: Num rows: 41 Data size: 11480 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3381,7 +3342,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3404,7 +3404,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3486,14 +3486,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3512,65 +3512,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3593,7 +3535,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3608,7 +3550,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3626,7 +3568,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3649,7 +3630,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3664,6 +3645,25 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 609 Data size: 113274 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3731,13 +3731,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 6 (BROADCAST_EDGE) + Vertex 5 <- Vertex 2 (BROADCAST_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -3757,7 +3757,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 44 Data size: 3784 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -3773,7 +3773,22 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 Map Operator Tree: TableScan alias: y @@ -3792,7 +3807,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -3812,7 +3827,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -3828,41 +3843,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 1316 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3879,7 +3860,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 41 Data size: 11480 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -3888,6 +3869,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/count.q.out ql/src/test/results/clientpositive/llap/count.q.out index 2953718..2e88b2b 100644 --- ql/src/test/results/clientpositive/llap/count.q.out +++ ql/src/test/results/clientpositive/llap/count.q.out @@ -42,10 +42,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -68,7 +68,7 @@ STAGE PLANS: value expressions: _col5 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -185,10 +185,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -205,7 +205,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -271,7 +271,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -315,10 +315,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -340,7 +340,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -384,11 +384,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -404,7 +404,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -418,7 +418,7 @@ STAGE PLANS: Map-reduce partition columns: a (type: int) Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: b (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -470,11 +470,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -490,7 +490,7 @@ STAGE PLANS: Statistics: Num rows: 19 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -503,7 +503,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: b (type: int) Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -555,11 +555,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -575,7 +575,7 @@ STAGE PLANS: Statistics: Num rows: 9 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -589,7 +589,7 @@ STAGE PLANS: Map-reduce partition columns: c (type: int) Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE value expressions: b (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -644,10 +644,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -664,7 +664,7 @@ STAGE PLANS: value expressions: b (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -719,10 +719,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -739,7 +739,7 @@ STAGE PLANS: value expressions: $f3 (type: int), $f4 (type: int), $f5 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -787,10 +787,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -807,7 +807,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -855,10 +855,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -873,7 +873,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/cross_join.q.out ql/src/test/results/clientpositive/llap/cross_join.q.out index ae3f9bf..3a0521e 100644 --- ql/src/test/results/clientpositive/llap/cross_join.q.out +++ ql/src/test/results/clientpositive/llap/cross_join.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -12,10 +12,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -30,19 +30,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -60,6 +48,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -67,7 +67,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,19 +99,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -129,6 +117,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -149,10 +149,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -171,7 +171,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src2 @@ -190,24 +208,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -215,7 +215,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -229,10 +229,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -249,7 +249,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 250000 Data size: 21750000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -260,7 +260,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -279,7 +279,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 @@ -293,10 +293,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -313,7 +313,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 250000 Data size: 21750000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -324,7 +324,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 @@ -356,10 +356,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -379,7 +379,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -390,7 +390,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src2 diff --git ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out index 2bc17cb..e9e92b2 100644 --- ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out @@ -28,7 +28,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -42,10 +42,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -60,22 +60,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -93,6 +78,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -100,7 +100,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -114,11 +114,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d1 @@ -138,7 +138,40 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d2 @@ -158,7 +191,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -173,39 +206,6 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5500 Data size: 120932 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -213,7 +213,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -233,12 +233,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -253,29 +253,28 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Vertex 2 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan - alias: d2 + alias: d1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) @@ -291,25 +290,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2500 Data size: 54560 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -330,7 +311,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -342,6 +323,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 6 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -349,8 +349,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -364,12 +364,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -384,34 +384,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -429,7 +402,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -450,7 +438,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 100 Data size: 1540 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -462,6 +450,18 @@ STAGE PLANS: sort order: Statistics: Num rows: 50 Data size: 770 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 6 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -469,7 +469,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -487,13 +487,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -514,45 +514,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d1 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -564,7 +526,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -582,7 +544,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -603,7 +584,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE Column stats: NONE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -615,6 +596,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 5 Data size: 51 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out index c865788..8308f82 100644 --- ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out @@ -28,7 +28,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -42,10 +42,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -62,7 +62,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 5000 Data size: 106120 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -73,7 +73,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -95,7 +95,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -109,11 +109,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) - Map 3 <- Map 1 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: d1 @@ -133,7 +133,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d2 @@ -161,7 +161,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: a @@ -178,7 +178,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 5500 Data size: 116432 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -196,7 +196,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -216,12 +216,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -238,7 +238,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Reducer 3 + 1 Vertex 3 Statistics: Num rows: 2500 Data size: 52560 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -249,7 +249,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -269,7 +269,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -283,7 +283,19 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Vertex 4 Map Operator Tree: TableScan alias: d2 @@ -302,18 +314,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator @@ -321,8 +321,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product -Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Vertex 2' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -336,12 +336,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -358,7 +358,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Reducer 3 + 1 Vertex 3 Statistics: Num rows: 25000 Data size: 655600 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -369,7 +369,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -386,7 +386,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 100 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -400,19 +400,7 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 1460 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d2 - Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -424,6 +412,18 @@ STAGE PLANS: sort order: Statistics: Num rows: 50 Data size: 730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) + Vertex 4 + Map Operator Tree: + TableScan + alias: d2 + Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -431,7 +431,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -449,12 +449,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 5 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -475,7 +475,32 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + input vertices: + 1 Vertex 4 + Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: d1 @@ -495,7 +520,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -509,7 +534,19 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 105 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Vertex 5 Map Operator Tree: TableScan alias: d2 @@ -528,43 +565,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - input vertices: - 1 Reducer 4 - Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1250 Data size: 26280 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 5 Data size: 47 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/ctas.q.out ql/src/test/results/clientpositive/llap/ctas.q.out index fe492e4..379fd86 100644 --- ql/src/test/results/clientpositive/llap/ctas.q.out +++ ql/src/test/results/clientpositive/llap/ctas.q.out @@ -30,11 +30,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -50,7 +50,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -65,7 +65,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -186,11 +186,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -206,7 +206,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -221,7 +221,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -342,11 +342,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -362,7 +362,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -377,7 +377,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -562,11 +562,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -582,7 +582,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -597,7 +597,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -720,11 +720,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -740,7 +740,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -755,7 +755,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/cte_3.q.out ql/src/test/results/clientpositive/llap/cte_3.q.out index b6cda54..3f08e23 100644 --- ql/src/test/results/clientpositive/llap/cte_3.q.out +++ ql/src/test/results/clientpositive/llap/cte_3.q.out @@ -13,15 +13,7 @@ Plan optimized by CBO. Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=85) - Output:["_col0"] - Filter Operator [FIL_4] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -67,15 +59,7 @@ Plan optimized by CBO. Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=85) - Output:["_col0"] - Filter Operator [FIL_4] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -121,27 +105,11 @@ Plan optimized by CBO. Stage-5 Dependency Collection{} Stage-4 - Map 2 llap - File Output Operator [FS_8] - table:{"name:":"default.q1"} - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_9] (rows=1 width=0) - predicate:(key = '5') - TableScan [TS_5] (rows=1 width=0) - default@q2,q2,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 2 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q2"} - Select Operator [SEL_2] (rows=2 width=85) - Output:["_col0"] - Filter Operator [FIL_4] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git ql/src/test/results/clientpositive/llap/cte_5.q.out ql/src/test/results/clientpositive/llap/cte_5.q.out index 58a963b..0d46093 100644 --- ql/src/test/results/clientpositive/llap/cte_5.q.out +++ ql/src/test/results/clientpositive/llap/cte_5.q.out @@ -61,7 +61,7 @@ src_thrift srcbucket srcbucket2 srcpart -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.colnum @@ -77,34 +77,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=2 width=89) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_13] (rows=2 width=89) - Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=1 width=3) - Filter Operator [FIL_11] (rows=1 width=3) - predicate:(UDFToDouble(colnum) = 5.0) - TableScan [TS_0] (rows=1 width=3) - mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=2 width=85) - Filter Operator [FIL_12] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: with q1 as (select * from src where key= '5') select a.colnum from mydb.q1 as a join q1 as b diff --git ql/src/test/results/clientpositive/llap/cte_mat_1.q.out ql/src/test/results/clientpositive/llap/cte_mat_1.q.out index a1156e8..760690a 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_1.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_1.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -14,30 +14,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=4 width=85) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) - Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=2 width=85) - Filter Operator [FIL_11] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=2 width=85) - Filter Operator [FIL_12] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 diff --git ql/src/test/results/clientpositive/llap/cte_mat_2.q.out ql/src/test/results/clientpositive/llap/cte_mat_2.q.out index a1156e8..760690a 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_2.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_2.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -14,30 +14,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=4 width=85) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) - Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=2 width=85) - Filter Operator [FIL_11] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=2 width=85) - Filter Operator [FIL_12] (rows=2 width=87) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 diff --git ql/src/test/results/clientpositive/llap/cte_mat_3.q.out ql/src/test/results/clientpositive/llap/cte_mat_3.q.out index cb88f0c..138a0b0 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_3.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_3.q.out @@ -13,46 +13,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=0) - predicate:key is not null - TableScan [TS_5] (rows=1 width=0) - default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git ql/src/test/results/clientpositive/llap/cte_mat_4.q.out ql/src/test/results/clientpositive/llap/cte_mat_4.q.out index 1c5be08..5ef2789 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_4.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_4.q.out @@ -52,46 +52,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=0) - predicate:key is not null - TableScan [TS_5] (rows=1 width=0) - default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -202,46 +173,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=0) - predicate:key is not null - TableScan [TS_5] (rows=1 width=0) - default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git ql/src/test/results/clientpositive/llap/cte_mat_5.q.out ql/src/test/results/clientpositive/llap/cte_mat_5.q.out index 0c22b72..2d63e89 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_5.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_5.q.out @@ -76,46 +76,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-3 Fetch Operator limit:-1 Stage-4 - Reducer 3 llap - File Output Operator [FS_15] - Merge Join Operator [MERGEJOIN_20] (rows=1 width=3) - Conds:RS_11.UDFToDouble(_col0)=RS_12.UDFToDouble(_col0)(Inner),Output:["_col0"] - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:UDFToDouble(_col0) - Select Operator [SEL_7] (rows=1 width=3) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1 width=3) - predicate:colnum is not null - TableScan [TS_5] (rows=1 width=3) - mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:UDFToDouble(_col0) - Select Operator [SEL_10] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=0) - predicate:key is not null - TableScan [TS_8] (rows=1 width=0) - default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Vertex 3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.q1"} - Select Operator [SEL_2] (rows=2 width=176) - Output:["_col0","_col1"] - Filter Operator [FIL_4] (rows=2 width=178) - predicate:(key = '5') - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 diff --git ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index ba14835..3faa445 100644 --- ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -105,34 +105,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=5 width=228) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_15] (rows=5 width=228) - Conds:RS_6._col1=RS_7._col3(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=5 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=5 width=88) - predicate:item is not null - TableScan [TS_0] (rows=5 width=88) - default@testdeci2,s,Tbl:COMPLETE,Col:COMPLETE,Output:["id","item"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col3 - Select Operator [SEL_5] (rows=1 width=312) - Output:["_col1","_col2","_col3"] - Filter Operator [FIL_14] (rows=1 width=312) - predicate:((id = 2) and item is not null) - TableScan [TS_3] (rows=1 width=312) - default@testdeci2,d,Tbl:COMPLETE,Col:COMPLETE,Output:["id","amount","sales_tax","item"] + Vertex 2 diff --git ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index ca03652..9de6ff4 100644 --- ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -99,7 +99,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -204,10 +204,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s @@ -226,7 +226,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 1b6bb1f..f059f7c 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -50,10 +50,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -74,7 +74,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -211,11 +211,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -232,7 +232,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -264,43 +298,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -340,11 +340,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -361,27 +361,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -400,7 +380,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -415,6 +395,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -467,11 +467,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -488,7 +488,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 day(_col0) (type: int) + 1 day(_col0) (type: int) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -520,43 +554,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: day(ds) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 day(_col0) (type: int) - 1 day(_col0) (type: int) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -596,11 +596,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -617,27 +617,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: day(_col0) (type: int) - sort order: + - Map-reduce partition columns: day(_col0) (type: int) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -656,7 +636,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -671,6 +651,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: day(_col0) (type: int) + sort order: + + Map-reduce partition columns: day(_col0) (type: int) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -712,12 +712,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -734,7 +734,57 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: srcpart_date @@ -766,10 +816,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: srcpart_hour @@ -801,59 +851,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -899,12 +899,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -922,47 +922,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -978,7 +938,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -997,7 +957,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1012,6 +972,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1066,11 +1066,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1086,7 +1086,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -1118,7 +1152,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -1133,43 +1167,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1209,11 +1209,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1230,27 +1230,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date_hour - filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1269,7 +1249,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1284,6 +1264,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date_hour + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1334,11 +1334,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1355,7 +1355,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -1387,43 +1421,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1463,11 +1463,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1484,27 +1484,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1523,7 +1503,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1538,6 +1518,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = 'I DONT EXIST') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1586,11 +1586,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1607,7 +1607,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -1639,43 +1673,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1715,11 +1715,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1736,7 +1736,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (UDFToDouble(_col0) * 2.0) (type: double) + 1 _col0 (type: double) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -1768,43 +1802,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (UDFToDouble(_col0) * 2.0) (type: double) - 1 _col0 (type: double) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1844,11 +1844,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1865,27 +1865,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_double_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1904,7 +1884,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1919,6 +1899,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_double_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1958,11 +1958,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -1979,27 +1979,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_double_hour - filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2018,7 +1998,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2033,11 +2013,31 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_double_hour + filterExpr: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hour) = 11.0) and hr is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: ListSink PREHOOK: query: select count(*) from srcpart join srcpart_double_hour on (srcpart.hr*2 = srcpart_double_hour.hr) where srcpart_double_hour.hour = 11 @@ -2085,11 +2085,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2106,7 +2106,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) + 1 UDFToString(_col0) (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -2138,43 +2172,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToString((UDFToDouble(hr) * 2.0)) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) - 1 UDFToString(_col0) (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2214,7 +2214,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' @@ -2228,12 +2228,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2246,27 +2246,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2285,7 +2265,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2300,7 +2280,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2320,7 +2320,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2347,7 +2347,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) @@ -2361,11 +2361,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2380,26 +2380,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_date_hour - filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) - Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2424,7 +2405,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2439,6 +2420,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_date_hour + filterExpr: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) + Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and (UDFToDouble(hour) = 11.0)) (type: boolean) + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string), hr (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2446,7 +2446,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2479,11 +2479,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2499,7 +2499,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -2531,7 +2565,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -2546,43 +2580,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2622,11 +2622,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2643,7 +2643,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -2675,43 +2709,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2732,11 +2732,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -2768,26 +2768,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2806,7 +2790,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2821,6 +2805,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2841,11 +2841,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2861,7 +2861,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_date @@ -2893,43 +2927,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2952,12 +2952,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -2974,7 +2974,57 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: srcpart_date @@ -3006,10 +3056,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: srcpart_hour @@ -3019,81 +3069,31 @@ STAGE PLANS: predicate: ((UDFToDouble(hour) = 11.0) and (UDFToDouble(hr) = 11.0)) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: hr (string) - Target Input: srcpart - Partition key expr: hr - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target column: hr (string) + Target Input: srcpart + Partition key expr: hr + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Target Vertex: Vertex 1 Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3135,14 +3135,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 2 (PARTITION_EDGE) + Vertex 7 <- Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3163,47 +3163,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart_hour - filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: hr (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3232,7 +3192,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3251,7 +3211,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3266,7 +3226,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3278,6 +3238,46 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_date + filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((date = '2008-04-08') and ds is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart_hour + filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hr (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3313,14 +3313,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 6 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3337,47 +3339,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3396,7 +3358,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3411,7 +3373,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3432,7 +3414,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3459,8 +3441,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 - Reducer 9 + Target Vertex: Vertex 1 + Vertex 8 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3481,8 +3483,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 6 - Vertex: Union 6 Stage: Stage-0 Fetch Operator @@ -3520,14 +3520,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 7 <- Union 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Union 6 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 6 + Vertex: Union 6 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3544,47 +3546,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3605,7 +3567,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3620,7 +3582,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3641,7 +3623,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3668,8 +3650,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 - Reducer 9 + Target Vertex: Vertex 1 + Vertex 8 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3690,8 +3692,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 6 - Vertex: Union 6 Stage: Stage-0 Fetch Operator @@ -3730,53 +3730,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Union 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE), Union 9 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 9 (CONTAINS) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 9 (CONTAINS), Vertex 11 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 9 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -3794,27 +3760,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3841,7 +3787,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -3856,8 +3802,28 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 5 - Reducer 12 + Target Vertex: Vertex 5 + Vertex 11 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3878,7 +3844,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3891,7 +3857,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3909,7 +3875,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: ds (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3922,7 +3906,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 736 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3943,10 +3947,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 - Union 9 - Vertex: Union 9 Stage: Stage-0 Fetch Operator @@ -3987,11 +3987,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4008,7 +4008,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4021,7 +4021,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4053,24 +4068,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4123,11 +4123,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4144,7 +4144,7 @@ STAGE PLANS: 0 day(_col0) (type: int) 1 day(_col0) (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4157,7 +4157,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4189,24 +4204,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: day(ds) Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4248,11 +4248,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4269,7 +4269,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -4278,7 +4278,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2420 Data size: 890560 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4291,7 +4291,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4323,10 +4338,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -4358,24 +4373,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4430,11 +4430,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4450,7 +4450,7 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col2 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 809600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4463,7 +4463,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -4495,7 +4510,7 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 @@ -4510,24 +4525,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4578,11 +4578,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4599,7 +4599,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4612,7 +4612,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -4644,24 +4659,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4701,11 +4701,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4722,7 +4722,7 @@ STAGE PLANS: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4735,7 +4735,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -4767,24 +4782,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: UDFToDouble(hr) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4824,11 +4824,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4845,7 +4845,7 @@ STAGE PLANS: 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -4858,7 +4858,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_double_hour @@ -4890,24 +4905,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: (UDFToDouble(hr) * 2.0) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -4947,7 +4947,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' @@ -4961,11 +4961,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 3 <- Vertex 1 (BROADCAST_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -4978,7 +4978,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart @@ -4998,7 +4998,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5015,7 +5015,7 @@ STAGE PLANS: 0 1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 500000 Data size: 11124000 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5026,7 +5026,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5048,7 +5048,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Vertex 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -5088,11 +5088,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5109,7 +5109,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5122,7 +5122,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -5154,24 +5169,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5192,11 +5192,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -5216,7 +5216,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5229,7 +5229,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -5245,21 +5260,6 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5280,11 +5280,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5300,7 +5300,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart_date @@ -5320,7 +5320,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5333,7 +5333,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5370,11 +5370,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5391,7 +5391,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1100 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -5400,7 +5400,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1210 Data size: 445280 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5413,7 +5413,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date @@ -5445,10 +5460,10 @@ STAGE PLANS: Target Input: srcpart Partition key expr: ds Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -5480,24 +5495,9 @@ STAGE PLANS: Target Input: srcpart Partition key expr: hr Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5539,11 +5539,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5564,7 +5564,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart_date @@ -5585,7 +5585,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -5594,7 +5594,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1 Data size: 444 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5607,7 +5607,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_hour @@ -5627,21 +5642,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -5677,13 +5677,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Map 1 (BROADCAST_EDGE), Union 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Vertex 1 (BROADCAST_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 4 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -5700,7 +5702,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: srcpart @@ -5720,27 +5722,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(ds) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5761,7 +5743,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5777,7 +5759,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -5789,7 +5771,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5804,7 +5786,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: ds + Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(ds) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5825,8 +5827,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -5900,11 +5900,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_orc @@ -5920,7 +5920,7 @@ STAGE PLANS: 0 _col0 (type: string), UDFToDouble(_col1) (type: double) 1 _col0 (type: string), UDFToDouble(_col2) (type: double) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 413600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -5933,7 +5933,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_date_hour @@ -5965,7 +5980,7 @@ STAGE PLANS: Target Input: srcpart_orc Partition key expr: ds Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Select Operator expressions: UDFToDouble(_col2) (type: double) outputColumnNames: _col0 @@ -5980,24 +5995,9 @@ STAGE PLANS: Target Input: srcpart_orc Partition key expr: UDFToDouble(hr) Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index 9dab93d..0c05a8f 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -156,12 +156,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -179,7 +179,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -199,7 +199,35 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d1 @@ -232,37 +260,9 @@ STAGE PLANS: Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -329,12 +329,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -352,7 +352,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -372,28 +372,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: d1 - filterExpr: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id (type: int), label (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -407,7 +386,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,6 +400,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: d1 + filterExpr: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((label) IN ('foo', 'bar') and id is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: int), label (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -479,10 +479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -500,7 +500,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) @@ -515,7 +515,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -574,7 +574,7 @@ bar baz baz baz -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -596,10 +596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -617,7 +617,7 @@ STAGE PLANS: 1 outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 51 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -628,7 +628,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: d1 @@ -651,7 +651,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -702,12 +702,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: agg @@ -725,7 +725,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col0 (type: decimal(10,0)) @@ -745,7 +745,35 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: d1 @@ -778,37 +806,9 @@ STAGE PLANS: Target Input: agg Partition key expr: dim_shops_id Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col2 (type: decimal(20,0)) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: decimal(20,0)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -867,11 +867,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: agg_01 @@ -889,7 +891,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -900,7 +902,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: dim_shops @@ -932,10 +934,10 @@ STAGE PLANS: Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: agg_01 @@ -953,7 +955,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -964,7 +966,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: dim_shops @@ -996,11 +998,9 @@ STAGE PLANS: Target Input: agg_01 Partition key expr: dim_shops_id Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1055,11 +1055,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1076,7 +1076,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2000000 Data size: 16000000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1089,7 +1089,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -1106,21 +1121,6 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out index cacde93..f91e9e6 100644 --- ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out @@ -115,11 +115,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -139,27 +139,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -178,7 +158,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -193,6 +173,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -232,13 +232,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -258,40 +258,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -310,7 +277,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -325,7 +292,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: all inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -376,11 +376,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -412,27 +412,10 @@ STAGE PLANS: Target Input: srcpart_small Partition key expr: ds Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 4 - Execution mode: llap - LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Vertex 4 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -451,7 +434,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -466,6 +449,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 360000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -505,11 +505,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -529,7 +529,43 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Statistics: Num rows: 27033 Data size: 2038454 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_small @@ -549,7 +585,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: alltypesorc_int @@ -569,42 +605,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - 2 _col0 (type: string) - Statistics: Num rows: 27033 Data size: 2038454 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -646,16 +646,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 6 (BROADCAST_EDGE) - Map 8 <- Reducer 4 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 5 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 7 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 4 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -688,73 +688,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: alltypesorc_int - filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cstring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -775,7 +709,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -790,7 +724,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -802,7 +736,53 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: all inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -814,7 +794,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -826,6 +806,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 8 + Map Operator Tree: + TableScan + alias: alltypesorc_int + filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -867,11 +867,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -891,27 +891,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: (key1 is not null and value1 is not null) (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key1 is not null and value1 is not null) (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string), value1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -930,7 +910,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -945,6 +925,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: (key1 is not null and value1 is not null) (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key1 is not null and value1 is not null) (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string), value1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1003,14 +1003,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE), Reducer 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1030,7 +1030,41 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: srcpart_small @@ -1066,51 +1100,17 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + LLAP IO: all inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1122,7 +1122,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1154,12 +1154,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1180,47 +1180,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: alltypesorc_int - filterExpr: cstring is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: cstring is not null (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cstring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1236,7 +1196,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1255,7 +1215,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1270,6 +1230,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: alltypesorc_int + filterExpr: cstring is not null (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: cstring is not null (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1311,16 +1311,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 7 (BROADCAST_EDGE) - Map 8 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 7 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 2 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 8 <- Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1341,60 +1341,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: alltypesorc_int - filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cstring (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1423,7 +1370,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1442,7 +1389,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1457,7 +1404,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1469,7 +1416,40 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: srcpart_small + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: all inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1481,6 +1461,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 8 + Map Operator Tree: + TableScan + alias: alltypesorc_int + filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1522,12 +1522,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1547,7 +1547,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1560,7 +1560,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_small @@ -1593,22 +1608,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1659,12 +1659,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart_date @@ -1684,7 +1684,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 404800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1697,7 +1697,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: srcpart_small @@ -1730,22 +1745,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index eaa394d..61b5460 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -166,10 +166,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -189,7 +189,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -239,11 +239,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -265,7 +265,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -281,7 +281,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -331,10 +331,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -354,7 +354,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -403,10 +403,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -426,7 +426,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -543,10 +543,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -566,7 +566,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -616,11 +616,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -642,7 +642,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -658,7 +658,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -708,10 +708,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -731,7 +731,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -780,10 +780,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -803,7 +803,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1322,10 +1322,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1344,7 +1344,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1394,10 +1394,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1417,7 +1417,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1467,11 +1467,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1488,7 +1488,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1511,7 +1511,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1561,10 +1561,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1584,7 +1584,7 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1639,10 +1639,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -1662,7 +1662,7 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2029,10 +2029,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -2052,7 +2052,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2101,10 +2101,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_orc @@ -2124,7 +2124,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2320,10 +2320,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc @@ -2341,7 +2341,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2538,10 +2538,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc @@ -2559,7 +2559,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out index 95a4e0f..9488338 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out @@ -123,10 +123,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -196,11 +196,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -222,7 +222,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -238,7 +238,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -288,10 +288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -311,7 +311,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -360,10 +360,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -383,7 +383,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -500,10 +500,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -523,7 +523,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -573,11 +573,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -599,7 +599,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -615,7 +615,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -665,10 +665,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -688,7 +688,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -737,10 +737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -760,7 +760,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1279,10 +1279,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1301,7 +1301,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1351,10 +1351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1374,7 +1374,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1424,11 +1424,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1445,7 +1445,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1468,7 +1468,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: tinyint) Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1518,10 +1518,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1541,7 +1541,7 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1596,10 +1596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -1619,7 +1619,7 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1986,10 +1986,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2009,7 +2009,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2058,10 +2058,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2081,7 +2081,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2432,10 +2432,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2455,7 +2455,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2506,10 +2506,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2529,7 +2529,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2580,10 +2580,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2603,7 +2603,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2654,10 +2654,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2677,7 +2677,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2728,10 +2728,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2751,7 +2751,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2802,10 +2802,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k @@ -2825,7 +2825,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2876,7 +2876,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1k diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index 41a7709..08e0352 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -83,10 +83,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -106,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -325,10 +325,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -348,7 +348,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -562,10 +562,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -585,7 +585,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -804,10 +804,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss @@ -826,7 +826,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1095,10 +1095,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss_orc @@ -1118,7 +1118,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1337,10 +1337,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: ss_orc @@ -1359,7 +1359,7 @@ STAGE PLANS: value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1605,10 +1605,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1631,7 +1631,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1735,10 +1735,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1761,7 +1761,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index b7679f1..a5c3055 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -85,10 +85,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -107,7 +107,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -177,10 +177,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -200,7 +200,7 @@ STAGE PLANS: value expressions: _col3 (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -367,10 +367,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -389,7 +389,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -459,10 +459,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -481,7 +481,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 347200 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -657,10 +657,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -679,7 +679,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -750,10 +750,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -773,7 +773,7 @@ STAGE PLANS: value expressions: _col4 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -873,10 +873,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -896,7 +896,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1070,10 +1070,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1092,7 +1092,7 @@ STAGE PLANS: Statistics: Num rows: 800 Data size: 15400 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1163,10 +1163,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1185,7 +1185,7 @@ STAGE PLANS: Statistics: Num rows: 1600 Data size: 556800 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1285,10 +1285,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1307,7 +1307,7 @@ STAGE PLANS: Statistics: Num rows: 1600 Data size: 422400 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1481,10 +1481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1504,7 +1504,7 @@ STAGE PLANS: value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1575,10 +1575,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: acid @@ -1598,7 +1598,7 @@ STAGE PLANS: value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/empty_join.q.out ql/src/test/results/clientpositive/llap/empty_join.q.out index 19aa89f..d927a1f 100644 --- ql/src/test/results/clientpositive/llap/empty_join.q.out +++ ql/src/test/results/clientpositive/llap/empty_join.q.out @@ -55,40 +55,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_14] - Map Join Operator [MAPJOIN_22] (rows=2 width=1) - Conds:SEL_2._col0=RS_10._col0(Left Outer),SEL_2._col0=RS_11._col0(Inner),Output:["_col0","_col1","_col2"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1 width=0) - predicate:id is not null - TableScan [TS_3] (rows=1 width=0) - default@test_2,t2,Tbl:PARTIAL,Col:NONE,Output:["id"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_11] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1 width=1) - Output:["_col0"] - Filter Operator [FIL_21] (rows=1 width=1) - predicate:id is not null - TableScan [TS_6] (rows=1 width=1) - default@test_3,t3,Tbl:COMPLETE,Col:NONE,Output:["id"] - <-Select Operator [SEL_2] (rows=1 width=1) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=1) - predicate:id is not null - TableScan [TS_0] (rows=1 width=1) - default@test_1,t1,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 1 PREHOOK: query: SELECT t1.id, t2.id, t3.id FROM test_1 t1 diff --git ql/src/test/results/clientpositive/llap/except_distinct.q.out ql/src/test/results/clientpositive/llap/except_distinct.q.out index 94125b5..10136f8 100644 --- ql/src/test/results/clientpositive/llap/except_distinct.q.out +++ ql/src/test/results/clientpositive/llap/except_distinct.q.out @@ -206,12 +206,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -234,30 +236,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -286,7 +265,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -309,7 +288,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -338,8 +340,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -368,16 +368,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 14 <- Map 13 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) - Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 7 (SIMPLE_EDGE) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Union 7 (CONTAINS), Vertex 13 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Union 5 (CONTAINS) + Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) + Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -400,53 +406,36 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), 1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: count(1) + aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: bigint) + Vertex 11 Map Operator Tree: TableScan alias: src @@ -469,7 +458,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -485,49 +474,43 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col2), sum(_col3) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), 1 (type: bigint), _col2 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint), (_col2 * _col3) (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 291 Data size: 56454 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: sum(_col2), sum(_col3) + aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 14 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -556,7 +539,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -585,7 +568,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -627,7 +610,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 145 Data size: 28130 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -669,7 +652,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 137 Data size: 26578 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -692,12 +675,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 - Union 5 - Vertex: Union 5 - Union 7 - Vertex: Union 7 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -726,12 +726,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -752,28 +754,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -807,7 +788,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -830,7 +811,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -864,8 +866,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index c3f8071..29812a6 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -53,13 +53,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 llap - File Output Operator [FS_3] - table:{"name:":"default.src_orc_merge_test_part"} - Select Operator [SEL_1] (rows=500 width=95) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 PREHOOK: query: insert overwrite table src_orc_merge_test_part partition(ds='2012-01-03', ts='2012-01-03+14:46:31') select * from src PREHOOK: type: QUERY @@ -78,7 +72,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -88,23 +82,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 llap - File Output Operator [FS_7] - table:{"name:":"default.src_orc_merge_test_part"} - Select Operator [SEL_6] (rows=100 width=95) - Output:["_col0","_col1"] - Limit [LIM_5] (rows=100 width=178) - Number of rows:100 - Select Operator [SEL_4] (rows=100 width=178) - Output:["_col0","_col1"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Limit [LIM_2] (rows=100 width=178) - Number of rows:100 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select count(1) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' PREHOOK: type: QUERY @@ -123,24 +101,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Group By Operator [GBY_6] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_2] (rows=500 width=102) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=102) - default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: alter table src_orc_merge_test_part partition (ds='2012-01-03', ts='2012-01-03+14:46:31') concatenate PREHOOK: type: ALTER_PARTITION_MERGE @@ -157,23 +124,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Group By Operator [GBY_6] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_2] (rows=500 width=102) - TableScan [TS_0] (rows=500 width=102) - default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31' PREHOOK: type: QUERY @@ -182,24 +139,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Group By Operator [GBY_6] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_2] (rows=500 width=102) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=102) - default@src_orc_merge_test_part,src_orc_merge_test_part,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: drop table src_orc_merge_test_part PREHOOK: type: DROPTABLE @@ -209,7 +155,7 @@ POSTHOOK: query: drop table src_orc_merge_test_part POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@src_orc_merge_test_part POSTHOOK: Output: default@src_orc_merge_test_part -Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( select src1.key as k1, src1.value as v1, @@ -233,42 +179,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_17] - Group By Operator [GBY_15] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_14] - Group By Operator [GBY_13] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(hash(_col0,_col1,_col2,_col3))"] - Select Operator [SEL_11] (rows=27556 width=356) - Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_20] (rows=27556 width=356) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - Select Operator [SEL_5] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_3] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_2] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key PREHOOK: type: QUERY @@ -277,27 +196,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_7] - Select Operator [SEL_5] (rows=10 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 2 PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x PREHOOK: type: QUERY @@ -306,37 +211,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_10] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=10 width=101) - Output:["_col0","_col1"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 3 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c PREHOOK: type: QUERY @@ -345,98 +227,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_42] - Select Operator [SEL_41] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_40] - Select Operator [SEL_38] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_37] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_34] (rows=3 width=16) - Output:["_col2","_col6"] - Filter Operator [FIL_33] (rows=3 width=16) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_52] (rows=3 width=16) - Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_29] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_50] (rows=18 width=84) - predicate:key is not null - TableScan [TS_27] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_25] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_51] (rows=1 width=101) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1 width=89) - Output:["_col0","_col1"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - Select Operator [SEL_17] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_16] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=2 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_49] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=2 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_48] (rows=5 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 7 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) cbo_t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p left outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int % c asc, cbo_t3.c_int desc PREHOOK: type: QUERY @@ -445,91 +249,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_39] - Select Operator [SEL_38] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] - Group By Operator [GBY_34] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] - PartitionCols:_col0, _col1 - Group By Operator [GBY_32] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 - Select Operator [SEL_31] (rows=1 width=16) - Output:["_col6","_col2"] - Filter Operator [FIL_30] (rows=1 width=16) - predicate:(((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) - Merge Join Operator [MERGEJOIN_48] (rows=3 width=16) - Conds:RS_27._col0=RS_28._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_46] (rows=18 width=84) - predicate:((c_int > 0) and key is not null) - TableScan [TS_24] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_22] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_47] (rows=1 width=101) - Conds:RS_19._col0=RS_20._col0(Left Outer),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_44] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1 width=89) - Output:["_col0","_col1"] - Group By Operator [GBY_16] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:key, c_int, c_float - Filter Operator [FIL_45] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 7 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b+c, a desc) cbo_t1 right outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 2) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c PREHOOK: type: QUERY @@ -538,76 +270,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_31] - Select Operator [SEL_30] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_29] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_27] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_26] (rows=1 width=20) - Output:["_col2","_col6"] - Filter Operator [FIL_25] (rows=1 width=20) - predicate:(((_col1 + _col4) >= 2) and ((_col1 > 0) or (_col6 >= 0))) - Merge Join Operator [MERGEJOIN_36] (rows=4 width=20) - Conds:RS_21._col0=RS_22._col0(Right Outer),RS_21._col0=RS_23._col0(Right Outer),Output:["_col1","_col2","_col4","_col6"] - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_19] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_34] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1 width=89) - Output:["_col0","_col1"] - Group By Operator [GBY_16] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:key, c_int, c_float - Filter Operator [FIL_35] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 5 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by c+a desc) cbo_t1 full outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by p+q desc, r asc) cbo_t2 on cbo_t1.a=p full outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int PREHOOK: type: QUERY @@ -616,86 +289,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 llap - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - Group By Operator [GBY_33] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0, _col1 - Group By Operator [GBY_31] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col6, _col2 - Select Operator [SEL_30] (rows=1 width=20) - Output:["_col6","_col2"] - Filter Operator [FIL_29] (rows=1 width=20) - predicate:(((_col1 + _col4) >= 0) and ((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) - Merge Join Operator [MERGEJOIN_42] (rows=4 width=20) - Conds:RS_25._col0=RS_26._col0(Outer),RS_25._col0=RS_27._col0(Right Outer),Output:["_col1","_col2","_col4","_col6"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=20 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=20 width=84) - predicate:(c_int > 0) - TableScan [TS_22] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_9] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_39] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1 width=89) - Output:["_col0","_col1"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - Select Operator [SEL_17] (rows=1 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_16] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_14] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_40] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0)) - TableScan [TS_11] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 6 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c PREHOOK: type: QUERY @@ -704,83 +310,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_33] - Select Operator [SEL_32] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_31] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0, _col1 - Group By Operator [GBY_29] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_28] (rows=3 width=16) - Output:["_col2","_col6"] - Filter Operator [FIL_27] (rows=3 width=16) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_43] (rows=3 width=16) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=18 width=84) - predicate:key is not null - TableScan [TS_21] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_19] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_42] (rows=1 width=101) - Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_6] (rows=1 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_39] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1 width=89) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_11] (rows=1 width=93) - Output:["_col0","_col1","_col2"],keys:key, c_int, c_float - Filter Operator [FIL_40] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null) - TableScan [TS_8] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 5 PREHOOK: query: explain select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc PREHOOK: type: QUERY @@ -807,62 +347,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) +Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_26] - Select Operator [SEL_25] (rows=3 width=87) - Output:["_col0"] - <-Union 3 [SIMPLE_EDGE] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_24] - Select Operator [SEL_5] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_1] (rows=20 width=80) - Output:["key"] - TableScan [TS_0] (rows=20 width=80) - default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 6 [CONTAINS] llap - Reduce Output Operator [RS_24] - Select Operator [SEL_12] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_8] (rows=20 width=80) - Output:["key"] - TableScan [TS_7] (rows=20 width=80) - default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 8 [CONTAINS] llap - Reduce Output Operator [RS_24] - Select Operator [SEL_21] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_20] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_19] - Group By Operator [GBY_18] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_17] (rows=20 width=80) - Output:["key"] - TableScan [TS_16] (rows=20 width=80) - default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from cbo_t3 s1 UNION ALL @@ -879,76 +373,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) +Vertex 9 <- Union 3 (CONTAINS), Vertex 8 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_31] - Select Operator [SEL_30] (rows=1 width=95) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - Group By Operator [GBY_27] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_5] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_1] (rows=20 width=80) - Output:["key"] - TableScan [TS_0] (rows=20 width=80) - default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 7 [CONTAINS] llap - Reduce Output Operator [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_12] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_8] (rows=20 width=80) - Output:["key"] - TableScan [TS_7] (rows=20 width=80) - default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 9 [CONTAINS] llap - Reduce Output Operator [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=1 width=95) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_21] (rows=1 width=87) - Output:["_col0"] - Group By Operator [GBY_20] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 8 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_19] - Group By Operator [GBY_18] (rows=1 width=8) - Output:["_col0"],aggregations:["count(key)"] - Select Operator [SEL_17] (rows=20 width=80) - Output:["key"] - TableScan [TS_16] (rows=20 width=80) - default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 5 PREHOOK: query: explain select cbo_t1.key from cbo_t1 join cbo_t3 where cbo_t1.key=cbo_t3.key and cbo_t1.key >= 1 PREHOOK: type: QUERY @@ -957,34 +392,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=18 width=85) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=6 width=85) - Output:["_col0"] - Filter Operator [FIL_13] (rows=6 width=85) - predicate:(UDFToDouble(key) >= 1.0) - TableScan [TS_0] (rows=20 width=80) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=6 width=85) - Output:["_col0"] - Filter Operator [FIL_14] (rows=6 width=85) - predicate:(UDFToDouble(key) >= 1.0) - TableScan [TS_3] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 left outer join cbo_t2 on cbo_t1.key=cbo_t2.key PREHOOK: type: QUERY @@ -993,32 +407,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=100 width=8) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_11] (rows=100 width=8) - Conds:RS_4._col0=RS_5._col0(Left Outer),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col0 - Select Operator [SEL_3] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 full outer join cbo_t2 on cbo_t1.key=cbo_t2.key PREHOOK: type: QUERY @@ -1027,32 +422,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=100 width=8) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_9] (rows=100 width=8) - Conds:RS_4._col0=RS_5._col0(Outer),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0 - Select Operator [SEL_1] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col0 - Select Operator [SEL_3] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select b, cbo_t1.c, cbo_t2.p, q, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1) cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key PREHOOK: type: QUERY @@ -1061,45 +437,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=291 width=101) - Output:["_col0","_col1","_col2","_col3","_col4"] - Merge Join Operator [MERGEJOIN_24] (rows=291 width=101) - Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=18 width=87) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_21] (rows=18 width=87) - predicate:key is not null - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=18 width=84) - predicate:key is not null - TableScan [TS_3] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=18 width=84) - predicate:key is not null - TableScan [TS_6] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select key, cbo_t1.c_int, cbo_t2.p, q from cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.key=p join (select key as a, c_int as b, cbo_t3.c_float as c from cbo_t3)cbo_t3 on cbo_t1.key=a PREHOOK: type: QUERY @@ -1108,45 +452,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=291 width=178) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_24] (rows=291 width=178) - Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col0","_col1","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_21] (rows=18 width=84) - predicate:key is not null - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18 width=80) - Output:["_col0"] - Filter Operator [FIL_22] (rows=18 width=80) - predicate:key is not null - TableScan [TS_3] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=18 width=84) - predicate:key is not null - TableScan [TS_6] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 full outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) PREHOOK: type: QUERY @@ -1155,55 +467,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_19] - Select Operator [SEL_18] (rows=36 width=101) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_17] (rows=36 width=101) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_28] (rows=36 width=101) - Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col6"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=18 width=84) - predicate:key is not null - TableScan [TS_11] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Filter Operator [FIL_9] (rows=10 width=182) - predicate:(((_col1 + _col4) = 2) and ((_col4 + 1) = 2)) - Merge Join Operator [MERGEJOIN_27] (rows=40 width=182) - Conds:RS_6._col0=RS_7._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=9 width=93) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_24] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=9 width=89) - Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_3] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 3 PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 right outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0) PREHOOK: type: QUERY @@ -1212,45 +483,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=50 width=101) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_12] (rows=50 width=101) - predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)) and ((_col4 + 1) = 2)) - Merge Join Operator [MERGEJOIN_19] (rows=200 width=101) - Conds:RS_8._col0=RS_9._col0(Right Outer),RS_8._col0=RS_10._col0(Right Outer),Output:["_col1","_col2","_col3","_col4","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_17] (rows=10 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=10 width=89) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=10 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_3] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_7] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_6] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 2 PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key order by x limit 1 PREHOOK: type: QUERY @@ -1259,34 +498,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Limit [LIM_9] (rows=1 width=97) - Number of rows:1 - Select Operator [SEL_8] (rows=10 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=10 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 3 PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x order by x,y limit 1 PREHOOK: type: QUERY @@ -1295,42 +514,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:1 Stage-1 - Reducer 4 llap - File Output Operator [FS_15] - Limit [LIM_14] (rows=1 width=20) - Number of rows:1 - Select Operator [SEL_13] (rows=5 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - Group By Operator [GBY_10] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0, _col1 - Group By Operator [GBY_8] (rows=5 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=10 width=101) - Output:["_col0","_col1"] - Group By Operator [GBY_4] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_2] (rows=10 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Select Operator [SEL_1] (rows=20 width=88) - Output:["key","c_int","c_float"] - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 4 PREHOOK: query: explain select key from(select key from (select key from cbo_t1 limit 5)cbo_t2 limit 5)cbo_t3 limit 5 PREHOOK: type: QUERY @@ -1339,37 +531,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 3 llap - File Output Operator [FS_13] - Limit [LIM_12] (rows=5 width=85) - Number of rows:5 - Limit [LIM_10] (rows=5 width=85) - Number of rows:5 - Select Operator [SEL_9] (rows=5 width=85) - Output:["_col0"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] - Limit [LIM_7] (rows=5 width=85) - Number of rows:5 - Limit [LIM_5] (rows=5 width=85) - Number of rows:5 - Select Operator [SEL_4] (rows=5 width=85) - Output:["_col0"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Limit [LIM_2] (rows=5 width=85) - Number of rows:5 - Select Operator [SEL_1] (rows=20 width=80) - Output:["_col0"] - TableScan [TS_0] (rows=20 width=80) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 3 PREHOOK: query: explain select key, c_int from(select key, c_int from (select key, c_int from cbo_t1 order by c_int limit 5)cbo_t1 order by c_int limit 5)cbo_t2 order by c_int limit 5 PREHOOK: type: QUERY @@ -1378,38 +547,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Limit [LIM_12] (rows=5 width=89) - Number of rows:5 - Select Operator [SEL_11] (rows=5 width=89) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Limit [LIM_8] (rows=5 width=89) - Number of rows:5 - Select Operator [SEL_7] (rows=5 width=89) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Limit [LIM_4] (rows=5 width=89) - Number of rows:5 - Select Operator [SEL_3] (rows=20 width=84) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=20 width=84) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Vertex 4 PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a limit 5) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc limit 5) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c limit 5 PREHOOK: type: QUERY @@ -1418,108 +564,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 7 llap - File Output Operator [FS_49] - Limit [LIM_48] (rows=1 width=20) - Number of rows:5 - Select Operator [SEL_47] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] - Select Operator [SEL_44] (rows=1 width=20) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_43] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0, _col1 - Group By Operator [GBY_41] (rows=1 width=20) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_40] (rows=3 width=16) - Output:["_col2","_col6"] - Filter Operator [FIL_39] (rows=3 width=16) - predicate:((_col1 > 0) or (_col6 >= 0)) - Merge Join Operator [MERGEJOIN_61] (rows=3 width=16) - Conds:RS_36._col0=RS_37._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=18 width=84) - Output:["_col0","_col1"] - Filter Operator [FIL_59] (rows=18 width=84) - predicate:key is not null - TableScan [TS_33] (rows=20 width=84) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=1 width=101) - predicate:((_col1 + _col4) >= 0) - Merge Join Operator [MERGEJOIN_60] (rows=2 width=101) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col0","_col1","_col2","_col4"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0 - Filter Operator [FIL_26] (rows=2 width=105) - predicate:_col0 is not null - Limit [LIM_24] (rows=3 width=105) - Number of rows:5 - Select Operator [SEL_23] (rows=3 width=105) - Output:["_col0","_col1"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - Select Operator [SEL_20] (rows=3 width=105) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_19] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_17] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_58] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_14] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Filter Operator [FIL_12] (rows=2 width=97) - predicate:_col0 is not null - Limit [LIM_10] (rows=3 width=97) - Number of rows:5 - Select Operator [SEL_9] (rows=3 width=97) - Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - Select Operator [SEL_6] (rows=3 width=97) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=3 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_56] (rows=6 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0))) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 7 PREHOOK: query: explain select cbo_t1.c_int from cbo_t1 left semi join cbo_t2 on cbo_t1.key=cbo_t2.key where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) PREHOOK: type: QUERY @@ -1528,38 +586,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=9 width=4) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_17] (rows=9 width=4) - Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=9 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=5 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=18 width=80) - Output:["_col0"] - Filter Operator [FIL_16] (rows=18 width=80) - predicate:key is not null - TableScan [TS_3] (rows=20 width=80) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select * from (select c, b, a from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 left semi join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where (b + 1 = 2) and (R.b > 0 or c >= 0) PREHOOK: type: QUERY @@ -1568,49 +601,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_18] - Select Operator [SEL_17] (rows=16 width=93) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_28] (rows=16 width=93) - Conds:RS_13._col0=RS_14._col0(Left Semi),RS_13._col0=RS_15._col0(Left Semi),Output:["_col0","_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=9 width=93) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_25] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=3 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=9 width=85) - Output:["_col0"] - Filter Operator [FIL_26] (rows=9 width=93) - predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null) - TableScan [TS_3] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=6 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_8] (rows=18 width=80) - Output:["_col0"] - Filter Operator [FIL_27] (rows=18 width=80) - predicate:key is not null - TableScan [TS_6] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select a, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) cbo_t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a PREHOOK: type: QUERY @@ -1619,92 +616,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 llap - File Output Operator [FS_41] - Select Operator [SEL_40] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] - Select Operator [SEL_38] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_37] (rows=1 width=101) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=1 width=101) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0 - Merge Join Operator [MERGEJOIN_51] (rows=1 width=93) - Conds:RS_30._col0=RS_31._col0(Left Semi),RS_30._col0=RS_32._col0(Left Semi),Output:["_col0","_col1"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=3 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_25] (rows=6 width=85) - Output:["_col0"] - Filter Operator [FIL_50] (rows=6 width=85) - predicate:(UDFToDouble(key) > 0.0) - TableScan [TS_23] (rows=20 width=80) - default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=1 width=93) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - Select Operator [SEL_8] (rows=1 width=101) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_7] (rows=1 width=101) - predicate:(((UDFToDouble(_col2) >= 1.0) or (_col3 >= 1)) and ((UDFToDouble(_col2) + UDFToDouble(_col3)) >= 0.0)) - Select Operator [SEL_6] (rows=1 width=101) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_5] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_3] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_48] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0)) and (UDFToDouble(key) > 0.0)) - TableScan [TS_0] (rows=20 width=88) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=1 width=85) - Output:["_col0"],keys:_col0 - Select Operator [SEL_21] (rows=1 width=85) - Output:["_col0"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - Select Operator [SEL_18] (rows=1 width=93) - Output:["_col0","_col1"] - Group By Operator [GBY_17] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_15] (rows=1 width=101) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float - Filter Operator [FIL_49] (rows=1 width=93) - predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and (c_float > 0) and ((c_int >= 1) or (c_float >= 1)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (UDFToDouble(key) > 0.0)) - TableScan [TS_12] (rows=20 width=88) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"] + Vertex 6 PREHOOK: query: explain select cbo_t1.key as x, c_int as c_int, (((c_int+c_float)*10)+5) as y from cbo_t1 PREHOOK: type: QUERY @@ -1756,33 +680,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_15] - Group By Operator [GBY_13] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_12] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count('2014')"] - Merge Join Operator [MERGEJOIN_18] (rows=400 width=8) - Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=20 width=88) - TableScan [TS_0] (rows=20 width=21) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=20 width=88) - TableScan [TS_3] (rows=20 width=21) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE + Vertex 3 PREHOOK: query: explain select * from src_cbo b @@ -1803,71 +708,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_29] - Select Operator [SEL_28] (rows=1 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_27] (rows=1 width=182) - predicate:_col3 is null - Merge Join Operator [MERGEJOIN_37] (rows=500 width=182) - Conds:RS_24._col1=RS_25._col0(Left Outer),Output:["_col0","_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col1 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=56 width=95) - Output:["_col0","_col1"] - Group By Operator [GBY_22] (rows=56 width=91) - Output:["_col0"],keys:_col1 - Select Operator [SEL_18] (rows=83 width=178) - Output:["_col1"] - Group By Operator [GBY_17] (rows=83 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=83 width=178) - Output:["_col0","_col1"],keys:_col2, _col0 - Select Operator [SEL_14] (rows=166 width=178) - Output:["_col2","_col0"] - Merge Join Operator [MERGEJOIN_36] (rows=166 width=178) - Conds:RS_11._col1=RS_12._col0(Inner),Output:["_col0","_col2"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col1 - Select Operator [SEL_4] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_34] (rows=166 width=178) - predicate:(value > 'val_2') - TableScan [TS_2] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_9] (rows=214 width=91) - Output:["_col0"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=214 width=91) - Output:["_col0"],keys:value - TableScan [TS_5] (rows=500 width=91) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] + Vertex 2 PREHOOK: query: explain select * from src_cbo b @@ -1890,75 +740,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_33] - Select Operator [SEL_32] (rows=1 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_31] (rows=1 width=182) - predicate:_col4 is null - Merge Join Operator [MERGEJOIN_41] (rows=250 width=182) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Left Outer),Output:["_col0","_col1","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_4] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0, _col1 - Group By Operator [GBY_2] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Select Operator [SEL_27] (rows=1 width=182) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_26] (rows=1 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=1 width=178) - Output:["_col0","_col1"],keys:_col2, _col3 - Merge Join Operator [MERGEJOIN_40] (rows=1 width=178) - Conds:RS_20._col0, _col1=RS_21._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0, _col1 - Select Operator [SEL_8] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_38] (rows=166 width=178) - predicate:(value > 'val_12') - TableScan [TS_6] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0, _col1 - Group By Operator [GBY_18] (rows=250 width=178) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_13] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1 - Group By Operator [GBY_11] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - TableScan [TS_9] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: create view cv1 as select * @@ -1991,59 +783,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_23] - Merge Join Operator [MERGEJOIN_33] (rows=1 width=178) - Conds:RS_19._col0, _col1=RS_20._col0, _col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"} - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=1 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Group By Operator [GBY_15] (rows=1 width=178) - Output:["_col0","_col1"],keys:_col2, _col3 - Merge Join Operator [MERGEJOIN_32] (rows=1 width=178) - Conds:RS_11._col0, _col1=RS_12._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Select Operator [SEL_4] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_30] (rows=166 width=178) - predicate:(value > 'val_9') - TableScan [TS_2] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - TableScan [TS_5] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"} + Vertex 2 PREHOOK: query: explain select * from (select * @@ -2066,59 +815,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_23] - Merge Join Operator [MERGEJOIN_33] (rows=1 width=178) - Conds:RS_19._col0, _col1=RS_20._col0, _col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=1 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Group By Operator [GBY_15] (rows=1 width=178) - Output:["_col0","_col1"],keys:_col2, _col3 - Merge Join Operator [MERGEJOIN_32] (rows=1 width=178) - Conds:RS_11._col0, _col1=RS_12._col0, _col1(Inner),Output:["_col2","_col3"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Select Operator [SEL_4] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_30] (rows=166 width=178) - predicate:(value > 'val_9') - TableScan [TS_2] (rows=500 width=178) - default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=250 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=250 width=178) - Output:["_col0","_col1"],keys:key, value - TableScan [TS_5] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from src_cbo @@ -2131,40 +837,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_14] - Merge Join Operator [MERGEJOIN_19] (rows=166 width=178) - Conds:RS_10._col0=RS_11._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=166 width=178) - predicate:(key > '9') - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=69 width=87) - Output:["_col0"],keys:KEY._col0 - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=69 width=87) - Output:["_col0"],keys:key - Filter Operator [FIL_18] (rows=166 width=87) - predicate:(key > '9') - TableScan [TS_3] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey @@ -2179,109 +859,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_46] - Select Operator [SEL_45] (rows=5 width=8) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_67] (rows=5 width=8) - Conds:RS_42._col1, _col4=RS_43._col0, _col1(Inner),Output:["_col0","_col3"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col1, _col4 - Merge Join Operator [MERGEJOIN_64] (rows=5 width=16) - Conds:RS_39._col0=RS_40._col1(Inner),Output:["_col0","_col1","_col3","_col4"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_40] - PartitionCols:_col1 - Select Operator [SEL_9] (rows=17 width=16) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_60] (rows=17 width=16) - predicate:((l_linenumber = 1) and l_partkey is not null) - TableScan [TS_7] (rows=100 width=16) - default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_5] (rows=50 width=4) - Output:["_col0"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0 - Group By Operator [GBY_3] (rows=50 width=4) - Output:["_col0"],keys:l_partkey - Filter Operator [FIL_59] (rows=100 width=4) - predicate:l_partkey is not null - TableScan [TS_0] (rows=100 width=4) - default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_43] - PartitionCols:_col0, _col1 - Group By Operator [GBY_37] (rows=4 width=8) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=4 width=8) - Output:["_col0","_col1"],keys:_col0, _col3 - Merge Join Operator [MERGEJOIN_66] (rows=14 width=8) - Conds:RS_31._col1=RS_32._col0(Inner),Output:["_col0","_col3"] - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=14 width=95) - Output:["_col0","_col1"] - Filter Operator [FIL_61] (rows=14 width=96) - predicate:(l_shipmode = 'AIR') - TableScan [TS_10] (rows=100 width=96) - default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=3 width=4) - Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=3 width=4) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_65] (rows=34 width=4) - Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col2"] - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=100 width=8) - Output:["_col0","_col1"] - Filter Operator [FIL_63] (rows=100 width=8) - predicate:l_partkey is not null - TableScan [TS_20] (rows=100 width=8) - default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey","l_linenumber"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Group By Operator [GBY_18] (rows=50 width=4) - Output:["_col0"],keys:KEY._col0 - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_17] - PartitionCols:_col0 - Group By Operator [GBY_16] (rows=50 width=4) - Output:["_col0"],keys:l_partkey - Filter Operator [FIL_62] (rows=100 width=4) - predicate:l_partkey is not null - TableScan [TS_13] (rows=100 width=4) - default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_partkey"] + Vertex 4 PREHOOK: query: explain select key, value, count(*) from src_cbo b @@ -2298,83 +889,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_36] - Merge Join Operator [MERGEJOIN_49] (rows=34 width=186) - Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col2 - Filter Operator [FIL_42] (rows=83 width=186) - predicate:_col2 is not null - Group By Operator [GBY_16] (rows=83 width=186) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_15] - PartitionCols:_col0, _col1 - Group By Operator [GBY_14] (rows=83 width=186) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1 - Merge Join Operator [MERGEJOIN_48] (rows=166 width=178) - Conds:RS_10._col0=RS_11._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_43] (rows=166 width=178) - predicate:(key > '8') - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=69 width=87) - Output:["_col0"],keys:KEY._col0 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=69 width=87) - Output:["_col0"],keys:key - Filter Operator [FIL_44] (rows=166 width=87) - predicate:(key > '8') - TableScan [TS_3] (rows=500 width=87) - default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=34 width=8) - Output:["_col0"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=34 width=8) - Output:["_col0"],keys:_col1 - Filter Operator [FIL_45] (rows=69 width=8) - predicate:_col1 is not null - Select Operator [SEL_47] (rows=69 width=8) - Output:["_col1"] - Group By Operator [GBY_24] (rows=69 width=95) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Group By Operator [GBY_22] (rows=69 width=95) - Output:["_col0","_col1"],aggregations:["count()"],keys:key - Filter Operator [FIL_46] (rows=166 width=87) - predicate:(key > '9') - TableScan [TS_19] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select p_mfgr, p_name, avg(p_size) from part @@ -2391,58 +917,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_23] - Merge Join Operator [MERGEJOIN_28] (rows=6 width=227) - Conds:RS_19._col1=RS_20._col0(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_6] (rows=13 width=227) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_5] (rows=13 width=227) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1 - Group By Operator [GBY_3] (rows=13 width=295) - Output:["_col0","_col1","_col2"],aggregations:["avg(p_size)"],keys:p_name, p_mfgr - Filter Operator [FIL_26] (rows=26 width=223) - predicate:p_name is not null - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=13 width=184) - Output:["_col0"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=13 width=184) - Output:["_col0"],keys:_col0 - Select Operator [SEL_11] (rows=26 width=491) - Output:["_col0"] - Filter Operator [FIL_27] (rows=26 width=491) - predicate:first_value_window_0 is not null - PTF Operator [PTF_10] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_9] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:p_mfgr - TableScan [TS_7] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 3 PREHOOK: query: explain select * from src_cbo @@ -2461,67 +945,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_26] - Select Operator [SEL_25] (rows=500 width=178) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - Select Operator [SEL_23] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=500 width=198) - predicate:((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) - Merge Join Operator [MERGEJOIN_31] (rows=500 width=198) - Conds:RS_19._col0=RS_20._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_30] (rows=500 width=194) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_16] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] - Group By Operator [GBY_7] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Group By Operator [GBY_5] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count()","count(key)"] - Filter Operator [FIL_28] (rows=166 width=87) - predicate:(key > '2') - TableScan [TS_2] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=69 width=91) - Output:["_col0","_col1"] - Group By Operator [GBY_14] (rows=69 width=87) - Output:["_col0"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=69 width=87) - Output:["_col0"],keys:key - Filter Operator [FIL_29] (rows=166 width=87) - predicate:(key > '2') - TableScan [TS_9] (rows=500 width=87) - default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select p_mfgr, b.p_name, p_size from part b @@ -2542,128 +976,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_54] - Select Operator [SEL_53] (rows=13 width=223) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_52] (rows=13 width=243) - predicate:CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END - Merge Join Operator [MERGEJOIN_76] (rows=26 width=243) - Conds:RS_49._col0, _col1=RS_50._col3, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_50] - PartitionCols:_col3, _col1 - Merge Join Operator [MERGEJOIN_75] (rows=2 width=223) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=4 width=223) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_34] (rows=4 width=219) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_33] - PartitionCols:_col0, _col1 - Group By Operator [GBY_32] (rows=4 width=219) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_74] (rows=8 width=219) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=8 width=219) - Output:["_col0","_col1"] - Filter Operator [FIL_69] (rows=8 width=223) - predicate:(p_size < 10) - TableScan [TS_19] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_26] (rows=5 width=98) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=5 width=98) - Output:["_col0"],keys:p_mfgr - TableScan [TS_22] (rows=26 width=98) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_43] - PartitionCols:_col0 - Group By Operator [GBY_40] (rows=13 width=121) - Output:["_col0"],keys:KEY._col0 - <-Map 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=13 width=121) - Output:["_col0"],keys:p_name - TableScan [TS_36] (rows=26 width=121) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_73] (rows=26 width=239) - Conds:RS_46._col1=RS_47._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] - PartitionCols:_col1 - Select Operator [SEL_1] (rows=26 width=223) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=26 width=223) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=2 width=114) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=2 width=114) - Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col1)"],keys:_col2 - Select Operator [SEL_14] (rows=8 width=219) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_72] (rows=8 width=219) - Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col1","_col2"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Select Operator [SEL_4] (rows=8 width=219) - Output:["_col0","_col1"] - Filter Operator [FIL_67] (rows=8 width=223) - predicate:(p_size < 10) - TableScan [TS_2] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_9] (rows=5 width=98) - Output:["_col0"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=5 width=98) - Output:["_col0"],keys:p_mfgr - TableScan [TS_5] (rows=26 width=98) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] + Vertex 3 PREHOOK: query: explain select p_name, p_size from @@ -2684,76 +1012,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (PARTITION_EDGE) +Vertex 8 <- Vertex 7 (PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_36] - Select Operator [SEL_35] (rows=26 width=125) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=26 width=125) - Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=26 width=145) - predicate:((_col2 = 0) or (_col5 is null and _col1 is not null and (_col3 >= _col2))) - Merge Join Operator [MERGEJOIN_42] (rows=26 width=145) - Conds:RS_29.UDFToDouble(_col1)=RS_30._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:UDFToDouble(_col1) - Merge Join Operator [MERGEJOIN_41] (rows=26 width=141) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_26] - Select Operator [SEL_1] (rows=26 width=125) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=26 width=125) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_27] - Group By Operator [GBY_12] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] - Group By Operator [GBY_7] (rows=1 width=8) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Group By Operator [GBY_5] (rows=1 width=76) - Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_38] (rows=8 width=4) - predicate:(p_size < 10) - TableScan [TS_2] (rows=26 width=4) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=1 width=12) - Output:["_col0","_col1"] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],keys:_col0 - Group By Operator [GBY_19] (rows=1 width=8) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 7 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_18] - Group By Operator [GBY_17] (rows=1 width=76) - Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_40] (rows=8 width=4) - predicate:(p_size < 10) - TableScan [TS_14] (rows=26 width=4) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] + Vertex 4 PREHOOK: query: explain select b.p_mfgr, min(p_retailprice) from part b @@ -2778,185 +1048,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 20 <- Reducer 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 12 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_83] - Select Operator [SEL_82] (rows=2 width=106) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_81] - Select Operator [SEL_80] (rows=2 width=106) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=2 width=126) - predicate:CASE WHEN ((_col3 = 0)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END - Merge Join Operator [MERGEJOIN_108] (rows=5 width=126) - Conds:RS_76._col0, _col1=RS_77._col3, _col1(Left Outer),Output:["_col0","_col1","_col3","_col4","_col7"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_77] - PartitionCols:_col3, _col1 - Merge Join Operator [MERGEJOIN_107] (rows=1 width=110) - Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=1 width=110) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_56] (rows=1 width=106) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col0, _col1 - Group By Operator [GBY_54] (rows=1 width=106) - Output:["_col0","_col1"],keys:_col0, _col3 - Merge Join Operator [MERGEJOIN_106] (rows=1 width=106) - Conds:RS_50._col1=RS_51._col0(Inner),Output:["_col0","_col3"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_50] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=1 width=114) - Output:["_col0","_col1"] - Filter Operator [FIL_98] (rows=1 width=114) - predicate:((_col2 - _col1) > 600.0) - Group By Operator [GBY_36] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - PartitionCols:_col0 - Group By Operator [GBY_34] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - TableScan [TS_32] (rows=26 width=106) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 20 [SIMPLE_EDGE] llap - SHUFFLE [RS_51] - PartitionCols:_col0 - Group By Operator [GBY_48] (rows=2 width=8) - Output:["_col0"],keys:KEY._col0 - <-Reducer 19 [SIMPLE_EDGE] llap - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_46] (rows=2 width=8) - Output:["_col0"],keys:_col1 - Select Operator [SEL_103] (rows=5 width=8) - Output:["_col1"] - Group By Operator [GBY_43] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0 - Group By Operator [GBY_41] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_40] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_39] (rows=26 width=106) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_70] - PartitionCols:_col0 - Group By Operator [GBY_67] (rows=5 width=98) - Output:["_col0"],keys:_col0 - Group By Operator [GBY_62] (rows=5 width=98) - Output:["_col0"],keys:KEY._col0 - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_61] - PartitionCols:_col0 - Group By Operator [GBY_60] (rows=5 width=98) - Output:["_col0"],keys:p_mfgr - TableScan [TS_58] (rows=26 width=98) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_76] - PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_105] (rows=5 width=122) - Conds:RS_73._col1=RS_74._col0(Left Outer),Output:["_col0","_col1","_col3","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_73] - PartitionCols:_col1 - Group By Operator [GBY_4] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0 - Group By Operator [GBY_2] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_1] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_0] (rows=26 width=106) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_74] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col0)"],keys:_col3 - Select Operator [SEL_27] (rows=1 width=106) - Output:["_col3","_col0"] - Merge Join Operator [MERGEJOIN_104] (rows=1 width=106) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col0","_col3"] - <-Reducer 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_22] (rows=2 width=8) - Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Group By Operator [GBY_20] (rows=2 width=8) - Output:["_col0"],keys:_col1 - Select Operator [SEL_102] (rows=5 width=8) - Output:["_col1"] - Group By Operator [GBY_17] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=5 width=106) - Output:["_col0","_col1"],aggregations:["min(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_14] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_13] (rows=26 width=106) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=1 width=114) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=1 width=114) - predicate:((_col2 - _col1) > 600.0) - Group By Operator [GBY_10] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_7] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_6] (rows=26 width=106) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] + Vertex 5 PREHOOK: query: explain select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1 PREHOOK: type: QUERY @@ -2965,25 +1078,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=20 width=52) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - PTF Operator [PTF_3] (rows=20 width=459) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] - Select Operator [SEL_2] (rows=20 width=459) - Output:["_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:0 - TableScan [TS_0] (rows=20 width=7) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] + Vertex 2 PREHOOK: query: explain select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1 PREHOOK: type: QUERY @@ -2992,25 +1093,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=20 width=52) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - PTF Operator [PTF_3] (rows=20 width=459) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"0 ASC NULLS FIRST","partition by:":"0"}] - Select Operator [SEL_2] (rows=20 width=459) - Output:["_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:0 - TableScan [TS_0] (rows=20 width=7) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int"] + Vertex 2 PREHOOK: query: explain select i, a, h, b, c, d, e, f, g, a as x, a +1 as y from (select max(c_int) over (partition by key order by value range UNBOUNDED PRECEDING) a, min(c_int) over (partition by key order by value range current row) b, count(c_int) over(partition by key order by value range 1 PRECEDING) c, avg(value) over (partition by key order by value range between unbounded preceding and unbounded following) d, sum(value) over (partition by key order by value range between unbounded preceding and current row) e, avg(c_float) over (partition by key order by value range between 1 preceding and unbounded following) f, sum(c_float) over (partition by key order by value range between 1 preceding and current row) g, max(c_float) over (partition by key order by value range between 1 preceding and unbounded following) h, min(c_float) over (partition by key order by value range between 1 preceding and 1 following) i from cbo_t1) cbo_t1 PREHOOK: type: QUERY @@ -3019,25 +1108,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=20 width=64) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - PTF Operator [PTF_3] (rows=20 width=621) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_2] (rows=20 width=621) - Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:key - TableScan [TS_0] (rows=20 width=169) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int","key","value"] + Vertex 2 PREHOOK: query: explain select *, rank() over(partition by key order by value) as rr from src1 PREHOOK: type: QUERY @@ -3046,25 +1123,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_4] (rows=25 width=179) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_3] (rows=25 width=443) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Select Operator [SEL_2] (rows=25 width=443) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:key - TableScan [TS_0] (rows=25 width=175) - default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3081,51 +1146,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_20] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] - Group By Operator [GBY_16] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_14] (rows=14 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_11] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Merge Join Operator [MERGEJOIN_25] (rows=60 width=86) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_24] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3142,51 +1171,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_20] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] - Group By Operator [GBY_16] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_14] (rows=14 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_11] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Merge Join Operator [MERGEJOIN_25] (rows=60 width=86) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_24] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3203,48 +1196,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_20] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_17] - Group By Operator [GBY_16] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_14] (rows=14 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_13] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col0 - Group By Operator [GBY_11] (rows=14 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Map Join Operator [MAPJOIN_25] (rows=60 width=86) - Conds:RS_6._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_24] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain select SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt)) @@ -3261,53 +1221,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_22] - Group By Operator [GBY_20] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_19] - Group By Operator [GBY_18] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(_col0)","sum(_col1)"] - Select Operator [SEL_16] (rows=12 width=94) - Output:["_col0","_col1"] - Group By Operator [GBY_15] (rows=12 width=94) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=12 width=94) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Merge Join Operator [MERGEJOIN_27] (rows=25 width=86) - Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=25 width=86) - Output:["_col0"] - Filter Operator [FIL_25] (rows=25 width=86) - predicate:key is not null - TableScan [TS_0] (rows=25 width=86) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=205 width=87) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_26] (rows=500 width=87) - predicate:key is not null - TableScan [TS_3] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: explain create table abcd (a int, b int, c int, d int) PREHOOK: type: CREATETABLE @@ -3340,25 +1262,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=2 width=19) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col2)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:_col0 - Group By Operator [GBY_2] (rows=4 width=19) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(DISTINCT b)","count(DISTINCT c)","sum(d)"],keys:a, b, c - Select Operator [SEL_1] (rows=4 width=19) - Output:["a","b","c","d"] - TableScan [TS_0] (rows=4 width=19) - default@abcd,abcd,Tbl:COMPLETE,Col:NONE,Output:["a","b","c","d"] + Vertex 2 PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a PREHOOK: type: QUERY @@ -3367,23 +1277,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=2 width=19) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT KEY._col1:0._col0)","count(DISTINCT KEY._col1:1._col0)","sum(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:a - Select Operator [SEL_1] (rows=4 width=19) - Output:["a","b","c","d"] - TableScan [TS_0] (rows=4 width=19) - default@abcd,abcd,Tbl:COMPLETE,Col:NONE,Output:["a","b","c","d"] + Vertex 2 PREHOOK: query: explain create table src_rc_merge_test(key int, value string) stored as rcfile PREHOOK: type: CREATETABLE @@ -3459,21 +1359,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test PREHOOK: type: QUERY @@ -3482,22 +1374,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: alter table tgt_rc_merge_test concatenate PREHOOK: type: ALTER_TABLE_MERGE @@ -3531,21 +1414,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test PREHOOK: type: QUERY @@ -3554,22 +1429,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_3] - Select Operator [SEL_1] (rows=5 width=6) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=5 width=6) - default@tgt_rc_merge_test,tgt_rc_merge_test,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: drop table src_rc_merge_test PREHOOK: type: DROPTABLE @@ -3594,27 +1460,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Merge Join Operator [MERGEJOIN_9] (rows=250000 width=87) - Conds:(Inner),Output:["_col0"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] - Select Operator [SEL_1] (rows=500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] - Select Operator [SEL_3] (rows=500 width=4) - TableScan [TS_2] (rows=500 width=10) - default@src,src2,Tbl:COMPLETE,Col:COMPLETE + Vertex 2 PREHOOK: query: explain create table nzhang_Tmp(a int, b string) PREHOOK: type: CREATETABLE @@ -3639,8 +1491,8 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -3650,25 +1502,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 3 llap - File Output Operator [FS_8] - table:{"name:":"default.nzhang_CTAS1"} - Limit [LIM_7] (rows=10 width=178) - Number of rows:10 - Select Operator [SEL_6] (rows=10 width=178) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Limit [LIM_4] (rows=10 width=178) - Number of rows:10 - Select Operator [SEL_3] (rows=500 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -3692,8 +1526,8 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -3703,25 +1537,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 3 llap - File Output Operator [FS_8] - table:{"name:":"default.nzhang_ctas3"} - Limit [LIM_7] (rows=10 width=192) - Number of rows:10 - Select Operator [SEL_6] (rows=10 width=192) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - Limit [LIM_4] (rows=10 width=192) - Number of rows:10 - Select Operator [SEL_3] (rows=500 width=192) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=192) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -3782,37 +1598,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=27556 width=356) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_15] (rows=27556 width=356) - Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Select Operator [SEL_2] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - Select Operator [SEL_5] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_3] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: CREATE TABLE myinput1(key int, value int) PREHOOK: type: CREATETABLE @@ -3837,28 +1630,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key PREHOOK: type: QUERY @@ -3867,39 +1645,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - Select Operator [SEL_10] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_21] (rows=6 width=9) - Conds:RS_4.key=RS_6.value(Inner),RS_4.key=RS_8.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:key - Filter Operator [FIL_18] (rows=3 width=8) - predicate:key is not null - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:value - Filter Operator [FIL_19] (rows=3 width=8) - predicate:value is not null - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:key - Filter Operator [FIL_20] (rows=3 width=8) - predicate:key is not null - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key PREHOOK: type: QUERY @@ -3908,33 +1660,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_9] (rows=6 width=9) - Conds:RS_3.key=RS_4.value(Inner),RS_3.key=RS_5.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value PREHOOK: type: QUERY @@ -3943,39 +1675,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - Select Operator [SEL_10] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_15] (rows=6 width=9) - Conds:RS_4.key, value=RS_6.value, key(Inner),RS_4.key, value=RS_8.key, value(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:key, value - Filter Operator [FIL_12] (rows=3 width=8) - predicate:value is not null - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:value, key - Filter Operator [FIL_13] (rows=3 width=8) - predicate:key is not null - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:key, value - Filter Operator [FIL_14] (rows=3 width=8) - predicate:value is not null - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value PREHOOK: type: QUERY @@ -3984,33 +1690,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=6 width=9) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_9] (rows=6 width=9) - Conds:RS_3.key, value=RS_4.value, key(Inner),RS_3.key, value=RS_5.key, value(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key, value - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:value, key - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key, value - TableScan [TS_2] (rows=3 width=8) - default@myinput1,c,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4019,28 +1705,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Left Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4049,28 +1720,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Right Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4079,28 +1735,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value PREHOOK: type: QUERY @@ -4109,28 +1750,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=3 width=9) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=3 width=9) - Conds:RS_2.key=RS_3.value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=3 width=8) - default@myinput1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:value - TableScan [TS_1] (rows=3 width=8) - default@myinput1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: CREATE TABLE smb_input(key int, value int) PREHOOK: type: CREATETABLE @@ -4205,28 +1831,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key AND a.value <=> b.value PREHOOK: type: QUERY @@ -4235,28 +1846,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key, value=RS_3.key, value(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key, value - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key, value - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input1 b ON a.key <=> b.key PREHOOK: type: QUERY @@ -4265,28 +1861,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Right Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key <=> b.key PREHOOK: type: QUERY @@ -4295,28 +1876,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Inner),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input1 b ON a.key <=> b.key PREHOOK: type: QUERY @@ -4325,28 +1891,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Less_than_equal_greater_than]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_6] - Select Operator [SEL_5] (rows=28 width=7) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_7] (rows=28 width=7) - Conds:RS_2.key=RS_3.key(Left Outer),Output:["_col0","_col1","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:key - TableScan [TS_0] (rows=26 width=7) - default@smb_input1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_1] (rows=26 width=7) - default@smb_input1,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 2 PREHOOK: query: drop table sales PREHOOK: type: DROPTABLE @@ -4409,36 +1960,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=15) - Conds:RS_8._col1=RS_9._col0(Left Semi),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1 width=13) - Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=1 width=13) - predicate:id is not null - TableScan [TS_0] (rows=1 width=13) - default@sales,sales,Tbl:COMPLETE,Col:NONE,Output:["name","id"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Group By Operator [GBY_7] (rows=2 width=14) - Output:["_col0"],keys:_col0 - Select Operator [SEL_5] (rows=2 width=14) - Output:["_col0"] - Filter Operator [FIL_16] (rows=2 width=14) - predicate:id is not null - TableScan [TS_3] (rows=2 width=14) - default@things,things,Tbl:COMPLETE,Col:NONE,Output:["id"] + Vertex 2 PREHOOK: query: drop table sales PREHOOK: type: DROPTABLE @@ -4463,42 +1991,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_16] - Map Join Operator [MAPJOIN_26] (rows=805 width=10) - Conds:MAPJOIN_25._col1=RS_13._col0(Inner),Output:["_col0"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=166 width=10) - Output:["_col0"] - Filter Operator [FIL_24] (rows=166 width=10) - predicate:(value > 'val_450') - TableScan [TS_6] (rows=500 width=10) - default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] - <-Map Join Operator [MAPJOIN_25] (rows=732 width=10) - Conds:SEL_2._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25 width=7) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=7) - predicate:key is not null - TableScan [TS_3] (rows=25 width=7) - default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=666 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=666 width=10) - predicate:((value > 'val_450') and key is not null) - TableScan [TS_0] (rows=2000 width=10) - default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' PREHOOK: type: QUERY @@ -4507,42 +2006,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_16] - Map Join Operator [MAPJOIN_26] (rows=805 width=10) - Conds:MAPJOIN_25._col1=RS_13._col0(Inner),Output:["_col0"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=166 width=10) - Output:["_col0"] - Filter Operator [FIL_24] (rows=166 width=10) - predicate:(value > 'val_450') - TableScan [TS_6] (rows=500 width=10) - default@src,src,Tbl:COMPLETE,Col:NONE,Output:["value"] - <-Map Join Operator [MAPJOIN_25] (rows=732 width=10) - Conds:SEL_2._col0=RS_10._col0(Inner),Output:["_col0","_col1"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25 width=7) - Output:["_col0"] - Filter Operator [FIL_23] (rows=25 width=7) - predicate:key is not null - TableScan [TS_3] (rows=25 width=7) - default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=666 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=666 width=10) - predicate:((value > 'val_450') and key is not null) - TableScan [TS_0] (rows=2000 width=10) - default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4567,33 +2037,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, @@ -4612,45 +2063,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Select Operator [SEL_14] (rows=29 width=227) - Output:["_col0","_col1","_col2","_col3"] - PTF Operator [PTF_13] (rows=29 width=223) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_12] (rows=29 width=223) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col2 - PTF Operator [PTF_10] (rows=29 width=223) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_9] (rows=29 width=223) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_21] (rows=29 width=223) - Conds:FIL_19.p_partkey=RS_5.p_partkey(Inner),Output:["_col1","_col2","_col5"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] - PartitionCols:p_partkey - Filter Operator [FIL_20] (rows=26 width=4) - predicate:p_partkey is not null - TableScan [TS_1] (rows=26 width=4) - default@part,p2,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] - <-Filter Operator [FIL_19] (rows=26 width=227) - predicate:p_partkey is not null - TableScan [TS_0] (rows=26 width=227) - default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey","p_name","p_mfgr","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4675,33 +2096,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4726,33 +2128,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_6] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4779,41 +2162,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_14] - Select Operator [SEL_12] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_11] (rows=26 width=223) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_8] (rows=26 width=223) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=26 width=223) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:rand() - Select Operator [SEL_4] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 4 PREHOOK: query: explain select abc.* @@ -4832,34 +2189,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - Map Join Operator [MAPJOIN_16] (rows=29 width=619) - Conds:FIL_14._col0=RS_8.p_partkey(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_8] - PartitionCols:p_partkey - Filter Operator [FIL_15] (rows=26 width=4) - predicate:p_partkey is not null - TableScan [TS_1] (rows=26 width=4) - default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] - <-Filter Operator [FIL_14] (rows=26 width=887) - predicate:_col0 is not null - PTF Operator [PTF_4] (rows=26 width=887) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=887) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=619) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_brand","p_comment","p_container","p_mfgr","p_name","p_partkey","p_retailprice","p_size","p_type"] + Vertex 2 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4878,35 +2214,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Select Operator [SEL_8] (rows=26 width=227) - Output:["_col0","_col1","_col2","_col3"] - PTF Operator [PTF_7] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}] - Select Operator [SEL_6] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2 - PTF Operator [PTF_4] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST, _col5 DESC NULLS LAST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - PTF Operator [PTF_1] (rows=26 width=223) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST, p_size DESC NULLS LAST","partition by:":"p_mfgr"}}] - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4929,35 +2244,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Select Operator [SEL_8] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_7] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_6] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2 - PTF Operator [PTF_4] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - PTF Operator [PTF_1] (rows=26 width=231) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"p_name ASC NULLS FIRST","partition by:":"p_mfgr"}}] - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size","p_retailprice"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -4980,33 +2274,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_9] - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select p_mfgr, p_name, p_size, @@ -5031,43 +2306,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Select Operator [SEL_11] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_10] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_9] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col2 - PTF Operator [PTF_7] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_6] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2 - PTF Operator [PTF_4] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 DESC NULLS LAST, _col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 4 PREHOOK: query: explain select p_mfgr, p_name, @@ -5098,33 +2345,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_10] - Select Operator [SEL_7] (rows=26 width=235) - Output:["_col0","_col1","_col2","_col3"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain select abc.p_mfgr, abc.p_name, @@ -5153,44 +2381,14 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_15] - Select Operator [SEL_13] (rows=29 width=259) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - PTF Operator [PTF_12] (rows=29 width=767) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_11] (rows=29 width=767) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_20] (rows=29 width=231) - Conds:FIL_18._col0=RS_8.p_partkey(Inner),Output:["_col1","_col2","_col5","_col7"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_8] - PartitionCols:p_partkey - Filter Operator [FIL_19] (rows=26 width=4) - predicate:p_partkey is not null - TableScan [TS_1] (rows=26 width=4) - default@part,p1,Tbl:COMPLETE,Col:COMPLETE,Output:["p_partkey"] - <-Filter Operator [FIL_18] (rows=26 width=503) - predicate:_col0 is not null - PTF Operator [PTF_4] (rows=26 width=503) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_3] (rows=26 width=503) - Output:["_col0","_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=235) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_partkey","p_retailprice","p_size"] + Vertex 3 PREHOOK: query: explain create view IF NOT EXISTS mfgr_price_view as select p_mfgr, p_brand, @@ -5292,10 +2490,10 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-4 Stats-Aggr Operator @@ -5305,49 +2503,8 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 3 llap - File Output Operator [FS_9] - table:{"name:":"default.part_4"} - Select Operator [SEL_7] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_6] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_5] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=499) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=499) - Output:["_col1","_col2","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=231) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_retailprice","p_size"] - Reducer 5 llap - File Output Operator [FS_20] - table:{"name:":"default.part_5"} - Select Operator [SEL_17] (rows=26 width=247) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - PTF Operator [PTF_16] (rows=26 width=499) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col2 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_15] (rows=26 width=499) - Output:["_col0","_col2","_col3","_col6"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - PartitionCols:_col2 - Select Operator [SEL_13] (rows=26 width=491) - Output:["_col1","_col2","_col5","sum_window_0"] - PTF Operator [PTF_12] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_11] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col2 - Please refer to the previous PTF Operator [PTF_3] + Vertex 3 + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 @@ -5392,43 +2549,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Select Operator [SEL_11] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_10] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}] - Select Operator [SEL_9] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_8] - PartitionCols:_col2, _col1 - PTF Operator [PTF_7] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - Select Operator [SEL_6] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:_col2, _col1 - PTF Operator [PTF_4] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noopwithmap","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 4 PREHOOK: query: explain select p_mfgr, p_name, @@ -5467,49 +2596,16 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_15] - Select Operator [SEL_13] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_12] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_11] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col2 - PTF Operator [PTF_9] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_8] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col2 - PTF Operator [PTF_6] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - Select Operator [SEL_5] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2, _col1 - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 5 PREHOOK: query: explain select p_mfgr, p_name, @@ -5544,41 +2640,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_12] - Select Operator [SEL_10] (rows=26 width=239) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - PTF Operator [PTF_9] (rows=26 width=491) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col2"}] - Select Operator [SEL_8] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col2 - PTF Operator [PTF_6] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST","partition by:":"_col2"}}] - Select Operator [SEL_5] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col2 - PTF Operator [PTF_3] (rows=26 width=491) - Function definitions:[{},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}},{"Partition table definition":{"name:":"noop","order by:":"_col2 ASC NULLS FIRST, _col1 ASC NULLS FIRST","partition by:":"_col2, _col1"}}] - Select Operator [SEL_2] (rows=26 width=491) - Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_1] - PartitionCols:p_mfgr, p_name - TableScan [TS_0] (rows=26 width=223) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_name","p_size"] + Vertex 4 PREHOOK: query: explain select distinct src.* from src PREHOOK: type: QUERY @@ -5587,29 +2657,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_7] - Group By Operator [GBY_5] (rows=500 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:_col0, _col1 - Group By Operator [GBY_3] (rows=500 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:rand() - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: explain select explode(array('a', 'b')) PREHOOK: type: QUERY @@ -5709,7 +2764,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -5719,31 +2774,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 llap - File Output Operator [FS_11] - table:{"name:":"default.dest_j1"} - Select Operator [SEL_9] (rows=1219 width=95) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_16] (rows=1219 width=178) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_14] (rows=500 width=87) - predicate:key is not null - TableScan [TS_0] (rows=500 width=87) - default@src,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,src2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 select src1.key, src2.value @@ -5772,43 +2803,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=3 width=33) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Map Join Operator [MAPJOIN_31] (rows=3 width=33) - Conds:FIL_27.key=RS_7.key(Inner),RS_7.key=RS_9.key(Inner),RS_9.key=RS_11.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:key - Filter Operator [FIL_28] (rows=1 width=30) - predicate:key is not null - TableScan [TS_1] (rows=1 width=30) - default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_9] - PartitionCols:key - Filter Operator [FIL_29] (rows=1 width=20) - predicate:key is not null - TableScan [TS_2] (rows=1 width=20) - default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_11] - PartitionCols:key - Filter Operator [FIL_30] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t4,d,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Filter Operator [FIL_27] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Vertex 1 PREHOOK: query: explain select /*+ STREAMTABLE(a,c) */ * @@ -5825,43 +2826,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_14] - Select Operator [SEL_13] (rows=3 width=33) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Map Join Operator [MAPJOIN_31] (rows=3 width=33) - Conds:FIL_27.key=RS_7.key(Inner),RS_7.key=RS_9.key(Inner),RS_9.key=RS_11.key(Inner),Output:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:key - Filter Operator [FIL_28] (rows=1 width=30) - predicate:key is not null - TableScan [TS_1] (rows=1 width=30) - default@t2,b,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_9] - PartitionCols:key - Filter Operator [FIL_29] (rows=1 width=20) - predicate:key is not null - TableScan [TS_2] (rows=1 width=20) - default@t3,c,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_11] - PartitionCols:key - Filter Operator [FIL_30] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t4,d,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Filter Operator [FIL_27] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Vertex 1 PREHOOK: query: explain FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -5870,38 +2841,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] - PartitionCols:rand() - Map Join Operator [MAPJOIN_18] (rows=550 width=87) - Conds:RS_3.UDFToDouble(key)=FIL_17.(key + 1)(Inner),Output:["_col0","_col1","_col5"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_3] - PartitionCols:UDFToDouble(key) - Filter Operator [FIL_16] (rows=1 width=30) - predicate:UDFToDouble(key) is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Filter Operator [FIL_17] (rows=500 width=87) - predicate:(key + 1) is not null - TableScan [TS_1] (rows=500 width=87) - default@src,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 4 PREHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -5931,34 +2879,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=1219 width=356) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: explain select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val PREHOOK: type: QUERY @@ -5967,38 +2894,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_13] - Group By Operator [GBY_11] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_10] - Group By Operator [GBY_9] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] - PartitionCols:rand() - Map Join Operator [MAPJOIN_18] (rows=1 width=33) - Conds:FIL_16.key=RS_5.val(Inner),Output:["_col0","_col6"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] - PartitionCols:val - Filter Operator [FIL_17] (rows=1 width=30) - predicate:val is not null - TableScan [TS_1] (rows=1 width=30) - default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["val"] - <-Filter Operator [FIL_16] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: explain select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key PREHOOK: type: QUERY @@ -6007,44 +2911,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Group By Operator [GBY_14] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_13] - Group By Operator [GBY_12] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - PartitionCols:rand() - Select Operator [SEL_9] (rows=1 width=33) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_21] (rows=1 width=33) - Conds:SEL_2._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1 width=30) - Output:["_col0","_col1"] - Filter Operator [FIL_20] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Select Operator [SEL_2] (rows=1 width=30) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: explain select count(1) from T1 a join T1 b on a.key = b.key PREHOOK: type: QUERY @@ -6053,42 +2928,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Group By Operator [GBY_14] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_13] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - PartitionCols:rand() - Map Join Operator [MAPJOIN_21] (rows=1 width=33) - Conds:SEL_2._col0=RS_7._col0(Inner) - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1 width=30) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1 width=30) - predicate:key is not null - TableScan [TS_3] (rows=1 width=30) - default@t1,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=1 width=30) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1 width=30) - predicate:key is not null - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: explain FROM T1 a LEFT OUTER JOIN T2 c ON c.key+1=a.key select sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -6097,40 +2945,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_9] - PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=33) - Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_17] (rows=1 width=33) - Conds:SEL_1.UDFToDouble(_col0)=RS_5.(UDFToDouble(_col0) + 1.0)(Left Outer),Output:["_col0","_col1","_col2"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] - PartitionCols:(UDFToDouble(_col0) + 1.0) - Select Operator [SEL_3] (rows=1 width=30) - Output:["_col0"] - TableScan [TS_2] (rows=1 width=30) - default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_1] (rows=1 width=30) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Vertex 3 PREHOOK: query: explain FROM T1 a RIGHT OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -6139,34 +2962,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE) -Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_11] - Group By Operator [GBY_9] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] - Group By Operator [GBY_7] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Map 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:rand() - Map Join Operator [MAPJOIN_14] (rows=1 width=33) - Conds:RS_2.UDFToDouble(key)=TS_1.(key + 1)(Right Outer),Output:["_col0","_col1","_col5"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_2] - PartitionCols:UDFToDouble(key) - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-TableScan [TS_1] (rows=1 width=30) - default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 4 PREHOOK: query: explain FROM T1 a FULL OUTER JOIN T2 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) PREHOOK: type: QUERY @@ -6175,37 +2979,15 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_11] - Group By Operator [GBY_9] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] - Group By Operator [GBY_7] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:rand() - Merge Join Operator [MERGEJOIN_12] (rows=1 width=33) - Conds:RS_2.UDFToDouble(key)=RS_3.(key + 1)(Outer),Output:["_col0","_col1","_col5"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - PartitionCols:UDFToDouble(key) - TableScan [TS_0] (rows=1 width=30) - default@t1,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:(key + 1) - TableScan [TS_1] (rows=1 width=30) - default@t2,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 4 PREHOOK: query: explain select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key PREHOOK: type: QUERY @@ -6214,32 +2996,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Hint]. Vertex dependency in root stage -Map 1 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Vertex 4 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_11] - Group By Operator [GBY_9] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_8] - Group By Operator [GBY_7] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:rand() - Map Join Operator [MAPJOIN_14] (rows=1 width=33) - Conds:TS_0.(key + 1)=RS_3.UDFToDouble(key)(Left Outer),Output:["_col0","_col6"] - <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_3] - PartitionCols:UDFToDouble(key) - TableScan [TS_1] (rows=1 width=30) - default@t1,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-TableScan [TS_0] (rows=1 width=30) - default@t1,k,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 diff --git ql/src/test/results/clientpositive/llap/explainuser_2.q.out ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 67f960a..e24ad94 100644 --- ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -177,51 +177,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=141 width=268) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_26] (rows=141 width=268) - Conds:RS_12._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=500 width=178) - predicate:key is not null - TableScan [TS_6] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_25] (rows=58 width=177) - Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col0","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500 width=369) - Output:["_col0"] - Filter Operator [FIL_22] (rows=500 width=91) - predicate:value is not null - TableScan [TS_0] (rows=500 width=91) - default@srcpart,z,Tbl:COMPLETE,Col:COMPLETE,Output:["value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 3 PREHOOK: query: EXPLAIN select @@ -282,143 +245,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 llap - File Output Operator [FS_55] - Limit [LIM_54] (rows=24 width=285) - Number of rows:100 - Select Operator [SEL_53] (rows=24 width=285) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_52] - Group By Operator [GBY_50] (rows=24 width=285) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_48] (rows=24 width=285) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2 - Select Operator [SEL_47] (rows=650 width=534) - Output:["_col12","_col20","_col2","_col13","_col21","_col3"] - Merge Join Operator [MERGEJOIN_97] (rows=650 width=534) - Conds:RS_44._col1, _col3=RS_45._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"] - <-Reducer 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_45] - PartitionCols:_col15, _col17 - Select Operator [SEL_40] (rows=190 width=447) - Output:["_col14","_col15","_col17","_col6","_col7"] - Merge Join Operator [MERGEJOIN_96] (rows=190 width=447) - Conds:RS_37._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"] - <-Reducer 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_37] - PartitionCols:_col6, _col4 - Merge Join Operator [MERGEJOIN_94] (rows=40 width=352) - Conds:RS_34._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - PartitionCols:_col1 - Select Operator [SEL_17] (rows=2 width=180) - Output:["_col1"] - Filter Operator [FIL_88] (rows=2 width=175) - predicate:((key = 'src1key') and value is not null) - TableScan [TS_15] (rows=25 width=175) - default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_34] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_93] (rows=40 width=352) - Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=2 width=178) - Output:["_col0"] - Filter Operator [FIL_87] (rows=2 width=178) - predicate:((value = 'd1value') and key is not null) - TableScan [TS_12] (rows=500 width=178) - default@src,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_92] (rows=40 width=352) - Conds:RS_28._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_29] - PartitionCols:_col3 - Select Operator [SEL_11] (rows=8 width=531) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_86] (rows=8 width=534) - predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null) - TableScan [TS_9] (rows=85 width=534) - default@ss,ss,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=10 width=185) - Output:["_col1"] - Filter Operator [FIL_85] (rows=10 width=178) - predicate:((key = 'srcpartkey') and value is not null) - TableScan [TS_6] (rows=2000 width=178) - default@srcpart,srcpart,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_38] - PartitionCols:_col4, _col2 - Merge Join Operator [MERGEJOIN_95] (rows=19 width=356) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] llap - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=8 width=531) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_89] (rows=8 width=534) - predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_18] (rows=85 width=534) - default@sr,sr,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"] - <-Map 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=500 width=178) - Output:["_col0"] - Filter Operator [FIL_90] (rows=500 width=178) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_21] (rows=500 width=178) - default@src,d2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_44] - PartitionCols:_col1, _col3 - Merge Join Operator [MERGEJOIN_91] (rows=414 width=269) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=170 width=356) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_83] (rows=170 width=356) - predicate:(v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_0] (rows=170 width=356) - default@cs,cs,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v2","k3","v3"] - <-Map 6 [SIMPLE_EDGE] llap - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0"] - Filter Operator [FIL_84] (rows=500 width=178) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_3] (rows=500 width=178) - default@src,d3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, z.value, y.value @@ -441,151 +282,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 6 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_59] - Group By Operator [GBY_57] (rows=28 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=28 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_51] (rows=73 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_85] (rows=73 width=177) - Conds:RS_48._col2=RS_49._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_81] (rows=500 width=87) - predicate:key is not null - TableScan [TS_42] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 14 [SIMPLE_EDGE] llap - SHUFFLE [RS_48] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_84] (rows=30 width=177) - Conds:RS_45._col1=RS_46._col1(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_46] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_39] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 13 [SIMPLE_EDGE] llap - SHUFFLE [RS_45] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=262 width=178) - Output:["_col1"] - Group By Operator [GBY_37] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_78] (rows=25 width=175) - predicate:value is not null - TableScan [TS_26] (rows=25 width=175) - Output:["key","value"] - <-Map 16 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=500 width=178) - predicate:value is not null - TableScan [TS_29] (rows=500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=28 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=73 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_83] (rows=73 width=177) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=87) - Output:["_col0"] - Filter Operator [FIL_77] (rows=500 width=87) - predicate:key is not null - TableScan [TS_16] (rows=500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_82] (rows=30 width=177) - Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=262 width=178) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25 width=175) - Output:["key","value"] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - Output:["key","value"] + Vertex 7 PREHOOK: query: explain SELECT x.key, y.value @@ -616,296 +329,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 10 <- Union 2 (CONTAINS) -Map 13 <- Union 14 (CONTAINS) -Map 20 <- Union 14 (CONTAINS) -Map 21 <- Union 16 (CONTAINS) -Map 24 <- Union 25 (CONTAINS) -Map 33 <- Union 25 (CONTAINS) -Map 34 <- Union 27 (CONTAINS) -Map 35 <- Union 29 (CONTAINS) -Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) -Reducer 17 <- Union 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS) -Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 30 <- Union 29 (SIMPLE_EDGE) -Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 10 <- Union 2 (CONTAINS) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 6 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 14 (CONTAINS) +Vertex 21 <- Union 16 (CONTAINS) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 8 (CONTAINS), Vertex 31 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 33 <- Union 25 (CONTAINS) +Vertex 34 <- Union 27 (CONTAINS) +Vertex 35 <- Union 29 (CONTAINS) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 llap - File Output Operator [FS_122] - Group By Operator [GBY_120] (rows=107 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 32 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=107 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=124 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_170] (rows=124 width=177) - Conds:RS_111._col2=RS_112._col0(Inner),Output:["_col2","_col5"] - <-Map 37 [SIMPLE_EDGE] llap - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_107] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=178) - predicate:key is not null - TableScan [TS_105] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] llap - SHUFFLE [RS_111] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_169] (rows=51 width=86) - Conds:RS_108._col1=RS_109._col1(Inner),Output:["_col2"] - <-Map 36 [SIMPLE_EDGE] llap - SHUFFLE [RS_109] - PartitionCols:_col1 - Select Operator [SEL_104] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_102] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 30 [SIMPLE_EDGE] llap - SHUFFLE [RS_108] - PartitionCols:_col1 - Select Operator [SEL_101] (rows=440 width=178) - Output:["_col1"] - Group By Operator [GBY_100] (rows=440 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 29 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=440 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_94] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=500 width=178) - predicate:value is not null - TableScan [TS_92] (rows=500 width=178) - Output:["key","value"] - <-Reducer 28 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=440 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_91] (rows=381 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_90] (rows=381 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 27 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_84] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=178) - predicate:value is not null - TableScan [TS_82] (rows=500 width=178) - Output:["key","value"] - <-Reducer 26 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_81] (rows=262 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_80] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 24 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_71] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=25 width=175) - predicate:value is not null - TableScan [TS_69] (rows=25 width=175) - Output:["key","value"] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_74] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=178) - predicate:value is not null - TableScan [TS_72] (rows=500 width=178) - Output:["key","value"] - <-Reducer 7 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=107 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=90 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=90 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=107 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_168] (rows=107 width=177) - Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 23 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=178) - predicate:key is not null - TableScan [TS_52] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 18 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_167] (rows=44 width=86) - Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=381 width=178) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 16 [SIMPLE_EDGE] - <-Map 21 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500 width=178) - predicate:value is not null - TableScan [TS_39] (rows=500 width=178) - Output:["key","value"] - <-Reducer 15 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=381 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=25 width=175) - predicate:value is not null - TableScan [TS_26] (rows=25 width=175) - Output:["key","value"] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=178) - predicate:value is not null - TableScan [TS_29] (rows=500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=90 width=177) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=73 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_166] (rows=73 width=177) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 12 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=178) - predicate:key is not null - TableScan [TS_16] (rows=500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_165] (rows=30 width=86) - Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_12] (rows=262 width=178) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25 width=175) - Output:["key","value"] - <-Map 10 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=262 width=178) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500 width=178) - Output:["key","value"] + Vertex 9 PREHOOK: query: EXPLAIN SELECT x.key, z.value, y.value @@ -920,44 +372,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=605 width=10) - Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_26] (rows=605 width=10) - Conds:MAPJOIN_25._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=500 width=10) - predicate:key is not null - TableScan [TS_6] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_25] (rows=550 width=10) - Conds:SEL_2._col0=RS_10._col1(Inner),Output:["_col0","_col3"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_10] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_3] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_2] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_22] (rows=500 width=10) - predicate:value is not null - TableScan [TS_0] (rows=500 width=10) - default@srcpart,z,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 1 PREHOOK: query: EXPLAIN select @@ -1018,118 +439,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 10 <- Map 9 (BROADCAST_EDGE) -Map 2 <- Map 1 (BROADCAST_EDGE) -Map 3 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (BROADCAST_EDGE) +Vertex 2 <- Vertex 1 (BROADCAST_EDGE) +Vertex 3 <- Vertex 10 (BROADCAST_EDGE), Vertex 2 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 llap - File Output Operator [FS_55] - Limit [LIM_54] (rows=100 width=10) - Number of rows:100 - Select Operator [SEL_53] (rows=805 width=10) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_52] - Group By Operator [GBY_50] (rows=805 width=10) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_49] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_48] (rows=1610 width=10) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2 - Select Operator [SEL_47] (rows=1610 width=10) - Output:["_col12","_col20","_col2","_col13","_col21","_col3"] - Map Join Operator [MAPJOIN_97] (rows=1610 width=10) - Conds:RS_44._col1, _col3=SEL_40._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"] - <-Map 2 [BROADCAST_EDGE] llap - BROADCAST [RS_44] - PartitionCols:_col1, _col3 - Map Join Operator [MAPJOIN_91] (rows=275 width=10) - Conds:RS_41._col0=SEL_5._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_41] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=170 width=34) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_83] (rows=170 width=34) - predicate:(v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_0] (rows=170 width=34) - default@cs,cs,Tbl:COMPLETE,Col:NONE,Output:["k1","v2","k3","v3"] - <-Select Operator [SEL_5] (rows=250 width=10) - Output:["_col0"] - Filter Operator [FIL_84] (rows=250 width=10) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_3] (rows=500 width=10) - default@src,d3,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_40] (rows=1464 width=10) - Output:["_col14","_col15","_col17","_col6","_col7"] - Map Join Operator [MAPJOIN_96] (rows=1464 width=10) - Conds:MAPJOIN_94._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_38] - PartitionCols:_col4, _col2 - Map Join Operator [MAPJOIN_95] (rows=275 width=10) - Conds:RS_24._col0=SEL_23._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 9 [BROADCAST_EDGE] llap - BROADCAST [RS_24] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=42 width=34) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_89] (rows=42 width=34) - predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null) - TableScan [TS_18] (rows=85 width=34) - default@sr,sr,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"] - <-Select Operator [SEL_23] (rows=250 width=10) - Output:["_col0"] - Filter Operator [FIL_90] (rows=250 width=10) - predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) - TableScan [TS_21] (rows=500 width=10) - default@src,d2,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_94] (rows=1331 width=10) - Conds:MAPJOIN_93._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 8 [BROADCAST_EDGE] llap - BROADCAST [RS_35] - PartitionCols:_col1 - Select Operator [SEL_17] (rows=12 width=7) - Output:["_col1"] - Filter Operator [FIL_88] (rows=12 width=7) - predicate:((key = 'src1key') and value is not null) - TableScan [TS_15] (rows=25 width=7) - default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_93] (rows=1210 width=10) - Conds:MAPJOIN_92._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 7 [BROADCAST_EDGE] llap - BROADCAST [RS_32] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=250 width=10) - Output:["_col0"] - Filter Operator [FIL_87] (rows=250 width=10) - predicate:((value = 'd1value') and key is not null) - TableScan [TS_12] (rows=500 width=10) - default@src,d1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_92] (rows=1100 width=10) - Conds:SEL_8._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"] - <-Map 6 [BROADCAST_EDGE] llap - BROADCAST [RS_29] - PartitionCols:_col3 - Select Operator [SEL_11] (rows=42 width=34) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_86] (rows=42 width=34) - predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null) - TableScan [TS_9] (rows=85 width=34) - default@ss,ss,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"] - <-Select Operator [SEL_8] (rows=1000 width=10) - Output:["_col1"] - Filter Operator [FIL_85] (rows=1000 width=10) - predicate:((key = 'srcpartkey') and value is not null) - TableScan [TS_6] (rows=2000 width=10) - default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, z.value, y.value @@ -1152,135 +472,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 10 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Map 9 <- Union 10 (CONTAINS) -Reducer 11 <- Map 13 (BROADCAST_EDGE), Map 14 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 3 <- Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 10 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 13 (BROADCAST_EDGE), Vertex 14 (BROADCAST_EDGE) +Vertex 12 <- Union 10 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) +Vertex 9 <- Union 10 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_59] - Group By Operator [GBY_57] (rows=550 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_51] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_85] (rows=550 width=10) - Conds:MAPJOIN_84._col2=RS_49._col0(Inner),Output:["_col1","_col2"] - <-Map 14 [BROADCAST_EDGE] llap - BROADCAST [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_81] (rows=500 width=10) - predicate:key is not null - TableScan [TS_42] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map Join Operator [MAPJOIN_84] (rows=288 width=10) - Conds:SEL_38._col1=RS_46._col1(Inner),Output:["_col1","_col2"] - <-Map 13 [BROADCAST_EDGE] llap - BROADCAST [RS_46] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_39] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_38] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 10 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Map 9 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_78] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Reducer 3 [CONTAINS] llap - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_83] (rows=550 width=10) - Conds:MAPJOIN_82._col2=RS_23._col0(Inner),Output:["_col1","_col2"] - <-Map 8 [BROADCAST_EDGE] llap - BROADCAST [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_77] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map Join Operator [MAPJOIN_82] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col1","_col2"] - <-Map 7 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, y.value @@ -1311,272 +515,29 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 17 <- Union 14 (CONTAINS) -Map 20 <- Union 21 (CONTAINS) -Map 27 <- Union 21 (CONTAINS) -Map 28 <- Union 23 (CONTAINS) -Map 29 <- Union 25 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) -Reducer 15 <- Map 18 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS) -Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS) -Reducer 26 <- Map 30 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE), Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 18 (BROADCAST_EDGE), Vertex 19 (BROADCAST_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 17 <- Union 14 (CONTAINS) +Vertex 20 <- Union 21 (CONTAINS) +Vertex 22 <- Union 21 (SORT_PARTITION_EDGE), Union 23 (CONTAINS) +Vertex 24 <- Union 23 (SORT_PARTITION_EDGE), Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 6 (CONTAINS), Vertex 30 (BROADCAST_EDGE), Vertex 31 (BROADCAST_EDGE) +Vertex 27 <- Union 21 (CONTAINS) +Vertex 28 <- Union 23 (CONTAINS) +Vertex 29 <- Union 25 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE), Union 6 (CONTAINS) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 llap - File Output Operator [FS_122] - Group By Operator [GBY_120] (rows=550 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 26 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_170] (rows=550 width=10) - Conds:MAPJOIN_169._col2=RS_112._col0(Inner),Output:["_col2","_col5"] - <-Map 31 [BROADCAST_EDGE] llap - BROADCAST [RS_112] - PartitionCols:_col0 - Select Operator [SEL_107] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=10) - predicate:key is not null - TableScan [TS_105] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_169] (rows=484 width=10) - Conds:SEL_101._col1=RS_109._col1(Inner),Output:["_col2"] - <-Map 30 [BROADCAST_EDGE] llap - BROADCAST [RS_109] - PartitionCols:_col1 - Select Operator [SEL_104] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_102] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_101] (rows=440 width=10) - Output:["_col1"] - Group By Operator [GBY_100] (rows=440 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 29 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_94] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=500 width=10) - predicate:value is not null - TableScan [TS_92] (rows=500 width=10) - Output:["key","value"] - <-Reducer 24 [CONTAINS] llap - Reduce Output Operator [RS_99] - PartitionCols:_col0, _col1 - Group By Operator [GBY_98] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_91] (rows=381 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_90] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 23 [SIMPLE_EDGE] - <-Map 28 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_84] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=10) - predicate:value is not null - TableScan [TS_82] (rows=500 width=10) - Output:["key","value"] - <-Reducer 22 [CONTAINS] llap - Reduce Output Operator [RS_89] - PartitionCols:_col0, _col1 - Group By Operator [GBY_88] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_81] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_80] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 21 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_71] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=25 width=7) - predicate:value is not null - TableScan [TS_69] (rows=25 width=7) - Output:["key","value"] - <-Map 27 [CONTAINS] llap - Reduce Output Operator [RS_79] - PartitionCols:_col0, _col1 - Group By Operator [GBY_78] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_74] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:value is not null - TableScan [TS_72] (rows=500 width=10) - Output:["key","value"] - <-Reducer 5 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=550 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_168] (rows=550 width=10) - Conds:MAPJOIN_167._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 19 [BROADCAST_EDGE] llap - BROADCAST [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_167] (rows=419 width=10) - Conds:SEL_48._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 18 [BROADCAST_EDGE] llap - BROADCAST [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_48] (rows=381 width=10) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 17 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 13 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Map 16 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Reducer 3 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1100 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_166] (rows=550 width=10) - Conds:MAPJOIN_165._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map Join Operator [MAPJOIN_165] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 9 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] + Vertex 7 PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -1722,22 +683,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242 width=10) - predicate:key is not null - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value @@ -1748,42 +694,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242 width=10) - predicate:value is not null - TableScan [TS_6] (rows=242 width=10) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: explain select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key @@ -1797,22 +714,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242 width=10) - predicate:key is not null - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key join tab2 s2 on s1.value=s2.value @@ -1823,42 +725,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242 width=10) - predicate:value is not null - TableScan [TS_6] (rows=242 width=10) - default@tab2,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: explain select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key @@ -1875,61 +748,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 6 <- Union 2 (CONTAINS) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (PARTITION_EDGE) +Vertex 6 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_26] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_39] (rows=558 width=10) - Conds:Union 2._col0=RS_19._col0(Inner) - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_36] (rows=500 width=10) - predicate:key is not null - TableScan [TS_15] (rows=500 width=10) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_37] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_34] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_33] (rows=242 width=10) - predicate:key is not null - TableScan [TS_0] (rows=242 width=10) - Output:["key"] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_18] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_35] (rows=242 width=10) - predicate:key is not null - TableScan [TS_10] (rows=242 width=10) - Output:["key"] + Vertex 4 PREHOOK: query: explain select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value @@ -1946,75 +774,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 8 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 8 <- Union 3 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 llap - File Output Operator [FS_32] - Group By Operator [GBY_30] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_29] - Group By Operator [GBY_28] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_51] (rows=587 width=10) - Conds:Union 3._col0=RS_25._col0(Inner) - <-Map 9 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=500 width=10) - Output:["_col0"] - Filter Operator [FIL_47] (rows=500 width=10) - predicate:key is not null - TableScan [TS_21] (rows=500 width=10) - default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] llap - Reduce Output Operator [RS_24] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_46] (rows=242 width=10) - predicate:key is not null - TableScan [TS_16] (rows=242 width=10) - Output:["key"] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_50] (rows=292 width=10) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_48] (rows=266 width=10) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242 width=10) - Output:["_col0"] - Filter Operator [FIL_44] (rows=242 width=10) - predicate:key is not null - TableScan [TS_3] (rows=242 width=10) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_43] (rows=242 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242 width=10) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242 width=10) - Output:["_col1"] - Filter Operator [FIL_45] (rows=242 width=10) - predicate:value is not null - TableScan [TS_6] (rows=242 width=10) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 5 PREHOOK: query: explain SELECT x.key, y.value @@ -2042,511 +811,34 @@ SELECT x.key, y.value FROM src1 x JOIN src1 y ON (x.key = y.key) JOIN (select key, value from src1 union all select key, value from src union all select key, value from src union all select key, value from src)z ON (x.value = z.value) POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 +Plan optimized by CBO. -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 12 <- Union 9 (CONTAINS) - Map 13 <- Union 9 (CONTAINS) - Map 16 <- Map 17 (BROADCAST_EDGE) - Map 18 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 19 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 20 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 21 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) - Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 17 - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 21 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 9 - Vertex: Union 9 +Vertex dependency in root stage +Vertex 1 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 10 <- Union 9 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 9 (CONTAINS) +Vertex 13 <- Union 9 (CONTAINS) +Vertex 16 <- Vertex 17 (BROADCAST_EDGE) +Vertex 18 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 19 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 20 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 21 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 8 <- Union 9 (CONTAINS) - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Union 4 + <-Vertex 11 [CONTAINS] + <-Vertex 18 [CONTAINS] + <-Vertex 19 [CONTAINS] + <-Vertex 20 [CONTAINS] + <-Vertex 21 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: explain SELECT x.key, y.value @@ -2577,285 +869,33 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 23 <- Map 24 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 23 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Union 15 (CONTAINS) +Vertex 16 <- Union 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 13 (CONTAINS) +Vertex 20 <- Union 15 (CONTAINS) +Vertex 23 <- Vertex 24 (BROADCAST_EDGE) +Vertex 25 <- Union 26 (CONTAINS) +Vertex 27 <- Union 26 (SORT_PARTITION_EDGE), Union 28 (CONTAINS) +Vertex 29 <- Union 28 (SORT_PARTITION_EDGE), Union 30 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (BROADCAST_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE), Union 7 (CONTAINS), Vertex 23 (BROADCAST_EDGE) +Vertex 32 <- Union 26 (CONTAINS) +Vertex 33 <- Union 28 (CONTAINS) +Vertex 34 <- Union 30 (CONTAINS) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 llap - File Output Operator [FS_122] - Group By Operator [GBY_120] (rows=530 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 31 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=484 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_167] (rows=484 width=10) - Conds:RS_111._col1=SEL_107._col1(Inner),Output:["_col0","_col3"] - <-Map 23 [BROADCAST_EDGE] llap - BROADCAST [RS_111] - PartitionCols:_col1 - Map Join Operator [MAPJOIN_166] (rows=27 width=7) - Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 24 [BROADCAST_EDGE] llap - BROADCAST [RS_109] - PartitionCols:_col0 - Select Operator [SEL_74] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=7) - predicate:key is not null - TableScan [TS_72] (rows=25 width=7) - default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_71] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_69] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_107] (rows=440 width=10) - Output:["_col1"] - Group By Operator [GBY_106] (rows=440 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_100] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=500 width=10) - predicate:value is not null - TableScan [TS_98] (rows=500 width=10) - Output:["key","value"] - <-Reducer 29 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_97] (rows=381 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_96] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_90] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:value is not null - TableScan [TS_88] (rows=500 width=10) - Output:["key","value"] - <-Reducer 27 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_87] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_86] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_77] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=25 width=7) - predicate:value is not null - TableScan [TS_75] (rows=25 width=7) - Output:["key","value"] - <-Map 32 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_80] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=500 width=10) - predicate:value is not null - TableScan [TS_78] (rows=500 width=10) - Output:["key","value"] - <-Reducer 6 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=577 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=605 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_165] (rows=605 width=10) - Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_164] (rows=550 width=10) - Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=500 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=500 width=10) - default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=381 width=10) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 15 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 14 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Map 19 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Reducer 4 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_163] (rows=550 width=10) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_162] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_149] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_147] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 9 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] + Vertex 8 PREHOOK: query: CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -2919,655 +959,50 @@ INSERT OVERWRITE TABLE a SELECT tmp.key, tmp.value INSERT OVERWRITE TABLE b SELECT tmp.key, tmp.value INSERT OVERWRITE TABLE c SELECT tmp.key, tmp.value POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-4 depends on stages: Stage-3 - Stage-0 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-4 - Stage-6 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-4 - Stage-7 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-3 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 12 <- Union 9 (CONTAINS) - Map 13 <- Union 9 (CONTAINS) - Map 16 <- Map 17 (BROADCAST_EDGE) - Map 18 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 19 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 20 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 21 <- Map 16 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) - Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 1 Map 17 - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col3 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 21 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - input vertices: - 0 Map 16 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 6 - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 577 Data size: 6053 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1127 Data size: 11896 Basic stats: COMPLETE Column stats: NONE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1239 Data size: 13085 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col4 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 634 Data size: 6658 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3550 Data size: 37482 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c - Union 2 - Vertex: Union 2 - Union 4 - Vertex: Union 4 - Union 9 - Vertex: Union 9 - - Stage: Stage-4 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - - Stage: Stage-5 - Stats-Aggr Operator - - Stage: Stage-1 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - - Stage: Stage-6 - Stats-Aggr Operator +Plan optimized by CBO. - Stage: Stage-2 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c +Vertex dependency in root stage +Vertex 1 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 10 <- Union 9 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 9 (CONTAINS) +Vertex 13 <- Union 9 (CONTAINS) +Vertex 16 <- Vertex 17 (BROADCAST_EDGE) +Vertex 18 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 19 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 20 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 21 <- Union 4 (CONTAINS), Vertex 16 (BROADCAST_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) +Vertex 8 <- Union 9 (CONTAINS) - Stage: Stage-7 - Stats-Aggr Operator +Stage-5 + Stats-Aggr Operator + Stage-0 + Move Operator + table:{"name:":"default.a"} + Stage-4 + Dependency Collection{} + Stage-3 + Union 4 + <-Vertex 11 [CONTAINS] + <-Vertex 18 [CONTAINS] + <-Vertex 19 [CONTAINS] + <-Vertex 20 [CONTAINS] + <-Vertex 21 [CONTAINS] + <-Vertex 3 [CONTAINS] +Stage-6 + Stats-Aggr Operator + Stage-1 + Move Operator + table:{"name:":"default.b"} + Please refer to the previous Stage-4 +Stage-7 + Stats-Aggr Operator + Stage-2 + Move Operator + table:{"name:":"default.c"} + Please refer to the previous Stage-4 PREHOOK: query: explain FROM @@ -3604,27 +1039,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 19 <- Union 13 (CONTAINS) -Map 20 <- Union 15 (CONTAINS) -Map 23 <- Map 24 (BROADCAST_EDGE) -Map 25 <- Union 26 (CONTAINS) -Map 32 <- Union 26 (CONTAINS) -Map 33 <- Union 28 (CONTAINS) -Map 34 <- Union 30 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Union 15 (SIMPLE_EDGE) -Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 31 <- Map 23 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Union 15 (CONTAINS) +Vertex 16 <- Union 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 13 (CONTAINS) +Vertex 20 <- Union 15 (CONTAINS) +Vertex 23 <- Vertex 24 (BROADCAST_EDGE) +Vertex 25 <- Union 26 (CONTAINS) +Vertex 27 <- Union 26 (SORT_PARTITION_EDGE), Union 28 (CONTAINS) +Vertex 29 <- Union 28 (SORT_PARTITION_EDGE), Union 30 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (BROADCAST_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE), Union 7 (CONTAINS), Vertex 23 (BROADCAST_EDGE) +Vertex 32 <- Union 26 (CONTAINS) +Vertex 33 <- Union 28 (CONTAINS) +Vertex 34 <- Union 30 (CONTAINS) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-5 Stats-Aggr Operator @@ -3634,266 +1069,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 - Reducer 8 llap - File Output Operator [FS_123] - table:{"name:":"default.a"} - Group By Operator [GBY_120] (rows=530 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 31 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_114] (rows=484 width=10) - Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_172] (rows=484 width=10) - Conds:RS_111._col1=SEL_107._col1(Inner),Output:["_col0","_col3"] - <-Map 23 [BROADCAST_EDGE] llap - BROADCAST [RS_111] - PartitionCols:_col1 - Map Join Operator [MAPJOIN_171] (rows=27 width=7) - Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 24 [BROADCAST_EDGE] llap - BROADCAST [RS_109] - PartitionCols:_col0 - Select Operator [SEL_74] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_162] (rows=25 width=7) - predicate:key is not null - TableScan [TS_72] (rows=25 width=7) - default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_71] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_69] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_107] (rows=440 width=10) - Output:["_col1"] - Group By Operator [GBY_106] (rows=440 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_100] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_166] (rows=500 width=10) - predicate:value is not null - TableScan [TS_98] (rows=500 width=10) - Output:["key","value"] - <-Reducer 29 [CONTAINS] llap - Reduce Output Operator [RS_105] - PartitionCols:_col0, _col1 - Group By Operator [GBY_104] (rows=881 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_97] (rows=381 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_96] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_90] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_165] (rows=500 width=10) - predicate:value is not null - TableScan [TS_88] (rows=500 width=10) - Output:["key","value"] - <-Reducer 27 [CONTAINS] llap - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1 - Group By Operator [GBY_94] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_87] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_86] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_77] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=7) - predicate:value is not null - TableScan [TS_75] (rows=25 width=7) - Output:["key","value"] - <-Map 32 [CONTAINS] llap - Reduce Output Operator [RS_85] - PartitionCols:_col0, _col1 - Group By Operator [GBY_84] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_80] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=10) - predicate:value is not null - TableScan [TS_78] (rows=500 width=10) - Output:["key","value"] - <-Reducer 6 [CONTAINS] llap - Reduce Output Operator [RS_119] - PartitionCols:_col0, _col1 - Group By Operator [GBY_118] (rows=1061 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Group By Operator [GBY_67] (rows=577 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_61] (rows=605 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_170] (rows=605 width=10) - Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"] - <-Map 22 [SIMPLE_EDGE] llap - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_160] (rows=500 width=10) - predicate:key is not null - TableScan [TS_52] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] llap - SHUFFLE [RS_58] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_169] (rows=550 width=10) - Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"] - <-Map 21 [SIMPLE_EDGE] llap - SHUFFLE [RS_56] - PartitionCols:_col1 - Select Operator [SEL_51] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=500 width=10) - predicate:(key is not null and value is not null) - TableScan [TS_49] (rows=500 width=10) - default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 16 [SIMPLE_EDGE] llap - SHUFFLE [RS_55] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=381 width=10) - Output:["_col1"] - Group By Operator [GBY_47] (rows=381 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 15 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_41] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=500 width=10) - predicate:value is not null - TableScan [TS_39] (rows=500 width=10) - Output:["key","value"] - <-Reducer 14 [CONTAINS] llap - Reduce Output Operator [RS_46] - PartitionCols:_col0, _col1 - Group By Operator [GBY_45] (rows=762 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_38] (rows=262 width=10) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_28] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=25 width=7) - predicate:value is not null - TableScan [TS_26] (rows=25 width=7) - Output:["key","value"] - <-Map 19 [CONTAINS] llap - Reduce Output Operator [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_31] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=500 width=10) - predicate:value is not null - TableScan [TS_29] (rows=500 width=10) - Output:["key","value"] - <-Reducer 4 [CONTAINS] llap - Reduce Output Operator [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=1155 width=10) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_25] (rows=550 width=10) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_168] (rows=550 width=10) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"] - <-Map 11 [SIMPLE_EDGE] llap - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=500 width=10) - predicate:key is not null - TableScan [TS_16] (rows=500 width=10) - default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] - PartitionCols:_col2 - Map Join Operator [MAPJOIN_167] (rows=288 width=10) - Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] - <-Map 10 [BROADCAST_EDGE] llap - BROADCAST [RS_20] - PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=25 width=7) - predicate:(key is not null and value is not null) - TableScan [TS_13] (rows=25 width=7) - default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_12] (rows=262 width=10) - Output:["_col1"] - Group By Operator [GBY_11] (rows=262 width=10) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_2] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=7) - predicate:value is not null - TableScan [TS_0] (rows=25 width=7) - Output:["key","value"] - <-Map 9 [CONTAINS] llap - Reduce Output Operator [RS_10] - PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=525 width=10) - Output:["_col0","_col1"],keys:_col1, _col0 - Select Operator [SEL_5] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:value is not null - TableScan [TS_3] (rows=500 width=10) - Output:["key","value"] - File Output Operator [FS_125] - table:{"name:":"default.b"} - Please refer to the previous Group By Operator [GBY_120] - File Output Operator [FS_127] - table:{"name:":"default.c"} - Please refer to the previous Group By Operator [GBY_120] + Vertex 8 Stage-6 Stats-Aggr Operator Stage-1 @@ -3940,10 +1116,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -3953,51 +1129,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 5 llap - File Output Operator [FS_21] - table:{"name:":"default.dest1"} - Group By Operator [GBY_19] (rows=1 width=96) - Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Group By Operator [GBY_14] (rows=1 width=272) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=1 width=272) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=1 width=272) - Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_6] (rows=1 width=272) - Output:["_col0","_col1"] - Group By Operator [GBY_5] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_1] (rows=500 width=10) - TableScan [TS_0] (rows=500 width=10) - default@src,s1,Tbl:COMPLETE,Col:COMPLETE - File Output Operator [FS_27] - table:{"name:":"default.dest2"} - Select Operator [SEL_26] (rows=1 width=456) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_25] (rows=1 width=464) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 - Please refer to the previous Group By Operator [GBY_14] + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 @@ -4012,33 +1144,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Unique_join]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=4400 width=10) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_9] (rows=4400 width=10) - Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_0] (rows=500 width=10) - default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] - PartitionCols:key - TableScan [TS_1] (rows=25 width=7) - default@src1,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key - TableScan [TS_2] (rows=2000 width=10) - default@srcpart,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: EXPLAIN SELECT @@ -4055,32 +1167,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_9] - Transform Operator [SCR_8] (rows=550 width=10) - command:cat - Merge Join Operator [MERGEJOIN_14] (rows=550 width=10) - Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_3] - PartitionCols:key - Filter Operator [FIL_12] (rows=500 width=10) - predicate:key is not null - TableScan [TS_0] (rows=500 width=10) - default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] - PartitionCols:key - Filter Operator [FIL_13] (rows=500 width=10) - predicate:key is not null - TableScan [TS_1] (rows=500 width=10) - default@src,b,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 2 PREHOOK: query: explain FROM ( @@ -4111,11 +1204,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Map 7 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) +Vertex 7 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -4125,67 +1218,8 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 llap - File Output Operator [FS_19] - table:{"name:":"default.dest1"} - Group By Operator [GBY_17] (rows=1 width=96) - Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_22] - PartitionCols:_col0, _col1 - Group By Operator [GBY_21] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_8] - <-Map 7 [CONTAINS] llap - Reduce Output Operator [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_12] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_11] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_22] - PartitionCols:_col0, _col1 - Group By Operator [GBY_21] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_12] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_6] (rows=1 width=272) - Output:["_col0","_col1"] - Group By Operator [GBY_5] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_1] (rows=500 width=10) - TableScan [TS_0] (rows=500 width=10) - default@src,s1,Tbl:COMPLETE,Col:COMPLETE - Reduce Output Operator [RS_22] - PartitionCols:_col0, _col1 - Group By Operator [GBY_21] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_6] - Reducer 5 llap - File Output Operator [FS_25] - table:{"name:":"default.dest2"} - Group By Operator [GBY_23] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + Vertex 4 + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 @@ -4216,10 +1250,10 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 6 <- Union 3 (CONTAINS) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) -Reducer 5 <- Union 3 (SIMPLE_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 3 (CONTAINS) Stage-4 Stats-Aggr Operator @@ -4229,53 +1263,8 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 llap - File Output Operator [FS_16] - table:{"name:":"default.dest1"} - Group By Operator [GBY_14] (rows=1 width=96) - Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_8] (rows=500 width=10) - Output:["_col0","_col1"] - TableScan [TS_7] (rows=500 width=10) - Output:["key","value"] - Reduce Output Operator [RS_19] - PartitionCols:_col0, _col1 - Group By Operator [GBY_18] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_8] - <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_6] (rows=1 width=272) - Output:["_col0","_col1"] - Group By Operator [GBY_5] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=1 width=8) - Output:["_col0"],aggregations:["count(1)"] - Select Operator [SEL_1] (rows=500 width=10) - TableScan [TS_0] (rows=500 width=10) - default@src,s1,Tbl:COMPLETE,Col:COMPLETE - Reduce Output Operator [RS_19] - PartitionCols:_col0, _col1 - Group By Operator [GBY_18] (rows=1 width=464) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_6] - Reducer 5 llap - File Output Operator [FS_22] - table:{"name:":"default.dest2"} - Group By Operator [GBY_20] (rows=1 width=280) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 - <- Please refer to the previous Union 3 [SIMPLE_EDGE] + Vertex 4 + Vertex 5 Stage-5 Stats-Aggr Operator Stage-1 diff --git ql/src/test/results/clientpositive/llap/explainuser_4.q.out ql/src/test/results/clientpositive/llap/explainuser_4.q.out index 9804569..7a1447a 100644 --- ql/src/test/results/clientpositive/llap/explainuser_4.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_4.q.out @@ -17,39 +17,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=2166 width=620) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_17] (rows=2166 width=620) - Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365 width=251) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=251) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1019 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1019 width=251) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=251) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -96,41 +71,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_19] (rows=2166 width=8) - Conds:RS_6._col0=RS_7._col0(Inner) - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365 width=2) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1365 width=2) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=2) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["cint"] - <-Map 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019 width=8) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1019 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 3 PREHOOK: query: select count(*) @@ -170,47 +118,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=615 width=12) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - Group By Operator [GBY_12] (rows=615 width=12) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=615 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Merge Join Operator [MERGEJOIN_21] (rows=2166 width=4) - Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365 width=5) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=1365 width=5) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=5) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"] - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019 width=8) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1019 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -256,39 +172,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=1501 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_17] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -335,41 +226,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Map Join Operator [MAPJOIN_19] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true - <-Map 4 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_18] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_17] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"] + Vertex 3 PREHOOK: query: select count(*) @@ -409,47 +273,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 llap - File Output Operator [FS_16] - Select Operator [SEL_15] (rows=750 width=215) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] - Group By Operator [GBY_12] (rows=750 width=215) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=1501 width=215) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Map Join Operator [MAPJOIN_21] (rows=1501 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365 width=215) - Output:["_col0"] - Filter Operator [FIL_20] (rows=1365 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365 width=215) - Output:["_col0","_col1"] - Filter Operator [FIL_19] (rows=1365 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -476,7 +308,7 @@ POSTHOOK: Input: default@alltypesorc -3799 1 10782 1 NULL 6 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from alltypesorc a left outer join alltypesorc b @@ -490,26 +322,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_8] - Merge Join Operator [MERGEJOIN_9] (rows=150994944 width=431) - Conds:(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],residual filter predicates:{((_col2 = _col14) or _col1 BETWEEN 1 AND 10)} - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_4] - Select Operator [SEL_1] (rows=12288 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - TableScan [TS_0] (rows=12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_5] - Select Operator [SEL_3] (rows=12288 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - TableScan [TS_2] (rows=12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 2 diff --git ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out index c5cdb77..2bea0ef 100644 --- ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out +++ ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: f @@ -117,7 +117,68 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [f] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col3 + Position of Big Table: 0 + Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: _col0 (type: int) + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col5 + Position of Big Table: 0 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col5 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: m @@ -194,7 +255,7 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [m] - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: g @@ -270,67 +331,6 @@ STAGE PLANS: name: default.filter_join_breaktask Truncated Path -> Alias: /filter_join_breaktask/ds=2008-04-08 [g] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col3 - Position of Big Table: 0 - Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 27 Data size: 452 Basic stats: COMPLETE Column stats: NONE - tag: 0 - value expressions: _col0 (type: int) - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col5 - Position of Big Table: 0 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col5 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 29 Data size: 497 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/groupby1.q.out ql/src/test/results/clientpositive/llap/groupby1.q.out index 0eecbb6..88c16c3 100644 --- ql/src/test/results/clientpositive/llap/groupby1.q.out +++ ql/src/test/results/clientpositive/llap/groupby1.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -44,7 +44,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -59,7 +59,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/groupby2.q.out ql/src/test/results/clientpositive/llap/groupby2.q.out index 29b85d1..d3ceed8 100644 --- ql/src/test/results/clientpositive/llap/groupby2.q.out +++ ql/src/test/results/clientpositive/llap/groupby2.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,7 +45,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -60,7 +60,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 100000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/groupby3.q.out ql/src/test/results/clientpositive/llap/groupby3.q.out index 3495de6..4a36689 100644 --- ql/src/test/results/clientpositive/llap/groupby3.q.out +++ ql/src/test/results/clientpositive/llap/groupby3.q.out @@ -43,11 +43,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -63,7 +63,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -75,7 +75,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double), _col1 (type: struct), _col2 (type: struct), _col3 (type: string), _col4 (type: string), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: struct) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/groupby_resolution.q.out ql/src/test/results/clientpositive/llap/groupby_resolution.q.out index f2a6ab0..4436ba1 100644 --- ql/src/test/results/clientpositive/llap/groupby_resolution.q.out +++ ql/src/test/results/clientpositive/llap/groupby_resolution.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -30,7 +30,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -66,10 +66,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -85,7 +85,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,11 +121,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -141,7 +141,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -156,7 +156,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -192,11 +192,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -212,7 +212,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -227,7 +227,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 47500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -263,10 +263,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -289,7 +289,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -325,10 +325,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -351,7 +351,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -387,11 +387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -414,7 +414,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -429,7 +429,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -465,11 +465,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -492,7 +492,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -507,7 +507,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -638,12 +638,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -665,7 +665,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -680,7 +680,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -695,7 +695,7 @@ STAGE PLANS: Map-reduce partition columns: 0 (type: int) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -785,10 +785,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -805,7 +805,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/having.q.out ql/src/test/results/clientpositive/llap/having.q.out index 267254c..7353c9a 100644 --- ql/src/test/results/clientpositive/llap/having.q.out +++ ql/src/test/results/clientpositive/llap/having.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -37,7 +37,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -102,10 +102,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -127,7 +127,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -479,10 +479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -505,7 +505,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -755,10 +755,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -780,7 +780,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -956,10 +956,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -982,7 +982,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1228,10 +1228,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1254,7 +1254,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index 33d88ce..8d6051d 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -32,11 +32,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -55,7 +55,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -68,7 +68,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -87,21 +102,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -153,11 +153,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -176,7 +176,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -189,7 +189,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -208,21 +223,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -272,11 +272,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -295,7 +295,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -308,7 +308,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -327,21 +342,6 @@ STAGE PLANS: Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -389,11 +389,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -412,7 +412,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -425,7 +425,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -444,21 +459,6 @@ STAGE PLANS: Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -506,11 +506,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -526,7 +526,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -539,7 +539,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -555,21 +570,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -617,11 +617,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -637,7 +637,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -650,7 +650,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -666,21 +681,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -760,11 +760,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -783,7 +783,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -796,7 +796,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -815,21 +830,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -881,11 +881,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -904,7 +904,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -918,7 +918,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -937,21 +952,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1003,11 +1003,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1026,7 +1026,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1039,7 +1039,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1058,21 +1073,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1124,11 +1124,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1147,7 +1147,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -1161,7 +1161,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1180,21 +1195,6 @@ STAGE PLANS: Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1253,7 +1253,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1273,10 +1273,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1296,7 +1296,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1311,7 +1311,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1336,7 +1336,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1451,7 +1451,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1471,10 +1471,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1494,7 +1494,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1509,7 +1509,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1534,7 +1534,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out index ee9930f..05adcbb 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_2.q.out @@ -35,11 +35,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -54,7 +54,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -71,8 +71,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -85,22 +85,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -115,6 +100,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -166,11 +166,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -185,7 +185,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -202,8 +202,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -216,22 +216,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -246,6 +231,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -308,11 +308,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -327,7 +327,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -346,9 +346,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -361,7 +361,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -376,7 +391,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -391,21 +406,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -461,11 +461,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -480,7 +480,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -499,9 +499,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -514,7 +514,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -529,7 +544,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -544,21 +559,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -629,14 +629,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -651,7 +653,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -666,7 +668,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -683,8 +685,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -697,7 +699,40 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -712,7 +747,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -727,7 +762,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -744,8 +779,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -758,40 +793,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -809,8 +811,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -879,14 +879,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 10 (BROADCAST_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 9 <- Union 4 (CONTAINS), Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -901,7 +903,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -916,7 +918,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -933,8 +935,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -947,7 +949,40 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -962,7 +997,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -977,7 +1012,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -994,8 +1029,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1008,40 +1043,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1059,8 +1061,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -1140,11 +1140,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1160,7 +1160,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1178,8 +1178,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1190,8 +1190,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1204,7 +1204,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1219,7 +1234,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1234,7 +1249,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1249,21 +1264,6 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1335,11 +1335,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1355,7 +1355,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1373,8 +1373,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1385,8 +1385,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1399,7 +1399,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1414,7 +1429,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1429,7 +1444,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1444,21 +1459,6 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out index 3375fa8..af6f03e 100644 --- ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out +++ ql/src/test/results/clientpositive/llap/identity_project_remove_skip.q.out @@ -25,11 +25,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,23 +45,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '105') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: '105' (type: string) - sort order: + - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -73,7 +57,7 @@ STAGE PLANS: 0 '105' (type: string) 1 '105' (type: string) input vertices: - 1 Reducer 4 + 1 Vertex 4 Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: '105' (type: string), 'val_105' (type: string) @@ -86,7 +70,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '105') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: '105' (type: string) + sort order: + + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/insert1.q.out ql/src/test/results/clientpositive/llap/insert1.q.out index aa09585..3ff7427 100644 --- ql/src/test/results/clientpositive/llap/insert1.q.out +++ ql/src/test/results/clientpositive/llap/insert1.q.out @@ -39,7 +39,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -93,7 +93,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -161,7 +161,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -215,7 +215,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -277,7 +277,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: insert2 diff --git ql/src/test/results/clientpositive/llap/insert_into1.q.out ql/src/test/results/clientpositive/llap/insert_into1.q.out index 5caefe8..8524fc2 100644 --- ql/src/test/results/clientpositive/llap/insert_into1.q.out +++ ql/src/test/results/clientpositive/llap/insert_into1.q.out @@ -25,7 +25,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -147,7 +147,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -269,7 +269,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/llap/intersect_all.q.out ql/src/test/results/clientpositive/llap/intersect_all.q.out index 4dac8a2..fa18977 100644 --- ql/src/test/results/clientpositive/llap/intersect_all.q.out +++ ql/src/test/results/clientpositive/llap/intersect_all.q.out @@ -146,35 +146,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -197,7 +176,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -218,7 +197,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -248,7 +227,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -269,8 +271,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 48500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -799,83 +799,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -898,7 +831,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -919,7 +852,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -940,7 +873,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -970,7 +903,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -991,7 +947,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1012,8 +991,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1542,12 +1542,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1568,28 +1570,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1615,7 +1596,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1645,7 +1626,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1671,8 +1673,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/intersect_distinct.q.out ql/src/test/results/clientpositive/llap/intersect_distinct.q.out index 3698f03..6ac445f 100644 --- ql/src/test/results/clientpositive/llap/intersect_distinct.q.out +++ ql/src/test/results/clientpositive/llap/intersect_distinct.q.out @@ -144,35 +144,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -195,7 +174,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -216,7 +195,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -239,7 +218,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -260,8 +262,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -599,83 +599,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -698,7 +631,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -719,7 +652,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -740,7 +673,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -763,7 +696,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -784,7 +740,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -805,8 +784,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1144,12 +1144,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1170,28 +1172,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: key - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: key (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1217,7 +1198,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1240,7 +1221,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1266,8 +1268,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/intersect_merge.q.out ql/src/test/results/clientpositive/llap/intersect_merge.q.out index 6217785..28658c0 100644 --- ql/src/test/results/clientpositive/llap/intersect_merge.q.out +++ ql/src/test/results/clientpositive/llap/intersect_merge.q.out @@ -43,38 +43,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 11 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -97,30 +76,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 11 Map Operator Tree: TableScan alias: b @@ -143,51 +120,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -208,7 +141,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -229,7 +162,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -252,28 +185,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -281,7 +216,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -294,36 +229,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) -PREHOOK: type: QUERY -POSTHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -346,7 +252,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -369,7 +296,36 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) +PREHOOK: type: QUERY +POSTHOOK: query: explain (select * from b intersect distinct select * from a) intersect distinct (select * from b intersect distinct select * from a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -392,30 +348,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -436,7 +369,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -457,7 +390,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -480,28 +413,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -509,7 +444,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -522,37 +457,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -575,30 +480,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 8 Execution mode: llap - LLAP IO: no inputs - Map 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 6 Data size: 18 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -621,7 +524,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct (select * from b intersect distinct (select * from a intersect distinct select * from b))) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -644,30 +577,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -688,7 +598,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -709,7 +642,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -730,7 +663,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -753,28 +686,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -782,7 +717,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -795,37 +730,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b -PREHOOK: type: QUERY -POSTHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -848,30 +753,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + Vertex 8 Execution mode: llap - LLAP IO: no inputs - Map 5 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -894,7 +797,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b +PREHOOK: type: QUERY +POSTHOOK: query: explain (((select * from b intersect distinct select * from a) intersect distinct select * from b) intersect distinct select * from a) intersect distinct select * from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -917,30 +850,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -961,72 +871,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col2 = 5) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1034,7 +902,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -1047,7 +915,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1068,60 +936,74 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col2 = 5) (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 7 Map Operator Tree: TableScan alias: b @@ -1144,7 +1026,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -1167,7 +1070,37 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect distinct select * from b) intersect distinct select * from a intersect distinct select * from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1190,30 +1123,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1234,7 +1144,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 12 + Vertex 11 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1255,7 +1188,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1276,7 +1209,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1299,28 +1232,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(_col2) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Reducer 8 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1328,7 +1263,7 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col2) keys: _col0 (type: int), _col1 (type: int) @@ -1341,35 +1276,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) -#### A masked pattern was here #### - Vertices: - Map 1 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -1392,7 +1299,28 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col2) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 7 Data size: 21 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Vertex 9 Map Operator Tree: TableScan alias: a @@ -1415,7 +1343,35 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from b intersect distinct (select * from a intersect all select * from b) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1438,7 +1394,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1459,7 +1415,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1482,7 +1438,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1503,7 +1482,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1524,8 +1526,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -1546,59 +1546,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1621,7 +1577,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1642,7 +1598,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1672,7 +1628,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1693,7 +1672,30 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1714,8 +1716,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -1736,60 +1736,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 7 (CONTAINS) - Reducer 8 <- Union 3 (CONTAINS), Union 7 (SIMPLE_EDGE) + Vertex 10 <- Union 7 (CONTAINS), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 7 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Union 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1812,7 +1770,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1833,7 +1791,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1854,7 +1812,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1884,7 +1842,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1905,7 +1886,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1939,10 +1920,29 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Union 3 - Vertex: Union 3 - Union 7 - Vertex: Union 7 + Vertex 9 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/join0.q.out ql/src/test/results/clientpositive/llap/join0.q.out index 1b23c3d..8238d94 100644 --- ql/src/test/results/clientpositive/llap/join0.q.out +++ ql/src/test/results/clientpositive/llap/join0.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM @@ -24,11 +24,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -46,25 +46,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -79,7 +61,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -93,6 +75,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -100,7 +100,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN FORMATTED SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM @@ -118,7 +118,7 @@ SELECT src1.key as k1, src1.value as v1, SORT BY k1, v1, k2, v2 POSTHOOK: type: QUERY #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM (SELECT * FROM src WHERE src.key < 10) src1 diff --git ql/src/test/results/clientpositive/llap/join1.q.out ql/src/test/results/clientpositive/llap/join1.q.out index d79a405..b4225ce 100644 --- ql/src/test/results/clientpositive/llap/join1.q.out +++ ql/src/test/results/clientpositive/llap/join1.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -47,27 +47,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -90,6 +70,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/join32_lessSize.q.out ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index 5c2e3d4..42dedd1 100644 --- ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -58,13 +58,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 25 + Estimated key counts: Vertex 3 => 25 keys: 0 _col0 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 58 Data size: 10266 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -131,7 +131,56 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 1 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: x @@ -207,7 +256,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: y @@ -283,55 +332,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 - Position of Big Table: 1 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 141 Data size: 37788 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -509,12 +509,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -531,13 +531,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 4 => 25 + Estimated key counts: Vertex 4 => 25 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Position of Big Table: 0 Statistics: Num rows: 44 Data size: 11616 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator @@ -602,7 +602,78 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3 + Position of Big Table: 1 + Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE + tag: 0 + value expressions: _col3 (type: string) + auto parallelism: true + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 0 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: z @@ -678,7 +749,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [z] - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: w @@ -753,7 +824,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [w] - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y @@ -829,77 +900,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3 - Position of Big Table: 1 - Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 102 Data size: 17850 Basic stats: COMPLETE Column stats: COMPLETE - tag: 0 - value expressions: _col3 (type: string) - auto parallelism: true - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 - Position of Big Table: 0 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 248 Data size: 65968 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j1 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j1 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -1073,11 +1073,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -1154,31 +1154,80 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - Estimated key counts: Map 4 => 25 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - input vertices: - 1 Map 4 - Position of Big Table: 0 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Position of Big Table: 0 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j2 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct dest_j2 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + Estimated key counts: Vertex 4 => 25 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + input vertices: + 1 Vertex 4 + Position of Big Table: 0 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -1246,7 +1295,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -1322,55 +1371,6 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Position of Big Table: 0 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j2 - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct dest_j2 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -1546,11 +1546,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: z @@ -1627,7 +1627,56 @@ STAGE PLANS: name: default.srcpart Truncated Path -> Alias: /srcpart/ds=2008-04-08/hr=11 [z] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Position of Big Table: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j2 + numFiles 1 + numRows 85 + rawDataSize 1600 + serialization.ddl struct dest_j2 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 1685 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: y @@ -1698,7 +1747,33 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [y] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + value expressions: _col0 (type: string) + auto parallelism: true + Vertex 5 Map Operator Tree: TableScan alias: x @@ -1774,81 +1849,6 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [x] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Position of Big Table: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value,val2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.dest_j2 - numFiles 1 - numRows 85 - rawDataSize 1600 - serialization.ddl struct dest_j2 { string key, string value, string val2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 1685 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - value expressions: _col0 (type: string) - auto parallelism: true Stage: Stage-2 Dependency Collection @@ -2036,11 +2036,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -2059,7 +2059,30 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 184500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + Vertex 3 Map Operator Tree: TableScan alias: y @@ -2079,7 +2102,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -2093,7 +2116,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2113,29 +2136,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 Stage: Stage-2 Dependency Collection @@ -2292,11 +2292,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -2315,7 +2315,30 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 184500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col3, _col4 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j2 + Vertex 3 Map Operator Tree: TableScan alias: y @@ -2335,7 +2358,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 10500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -2349,7 +2372,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: x @@ -2369,29 +2392,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 140 Data size: 37240 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_j2 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/join46.q.out ql/src/test/results/clientpositive/llap/join46.q.out index 56f6862..d21b449 100644 --- ql/src/test/results/clientpositive/llap/join46.q.out +++ ql/src/test/results/clientpositive/llap/join46.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -75,24 +75,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -110,6 +93,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -162,10 +162,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -182,27 +182,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -223,6 +203,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -254,7 +254,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 99 2 Mat NULL NULL NULL 101 2 Car 102 2 Del -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -276,10 +276,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -294,25 +294,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -333,6 +315,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN 100 AND 102 (type: boolean) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -340,7 +340,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102 @@ -382,10 +382,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -402,24 +402,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -437,6 +420,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -464,7 +464,7 @@ NULL NULL NULL 105 NULL None 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -484,10 +484,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -502,22 +502,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -538,6 +523,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -545,7 +545,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) @@ -572,7 +572,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -596,10 +596,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -614,22 +614,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -648,6 +633,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -655,7 +655,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -687,7 +687,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -709,10 +709,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -727,22 +727,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -761,6 +746,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -768,7 +768,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -798,7 +798,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -820,10 +820,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -838,22 +838,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -872,6 +857,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -879,7 +879,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -927,10 +927,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -947,24 +947,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -983,6 +966,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1015,7 +1015,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat 102 2 Del 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT col_1, col_2 FROM test1 LEFT OUTER JOIN test2 @@ -1037,10 +1037,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1055,22 +1055,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1093,6 +1078,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1100,7 +1100,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT col_1, col_2 FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -1125,7 +1125,7 @@ Mat Ema Bob NULL Car Del Car Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1149,10 +1149,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1167,22 +1167,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1201,6 +1186,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1208,7 +1208,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1240,7 +1240,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1262,10 +1262,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1280,22 +1280,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1314,6 +1299,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1321,7 +1321,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1348,7 +1348,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1370,10 +1370,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1388,22 +1388,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1422,6 +1407,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1429,7 +1429,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1479,10 +1479,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1499,24 +1499,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1535,6 +1518,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1565,7 +1565,7 @@ NULL NULL NULL 105 NULL None 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1589,10 +1589,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1607,22 +1607,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1641,6 +1626,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1648,7 +1648,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1680,7 +1680,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1702,10 +1702,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1720,22 +1720,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1754,6 +1739,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1761,7 +1761,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1791,7 +1791,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1813,10 +1813,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1831,22 +1831,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1865,6 +1850,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1872,7 +1872,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1922,10 +1922,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1942,24 +1942,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1978,6 +1961,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2012,7 +2012,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -2062,46 +2062,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: test1 - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -2118,24 +2084,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2151,7 +2100,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2170,7 +2119,41 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 4 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: test1 + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 6 Data size: 56 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2186,6 +2169,23 @@ STAGE PLANS: sort order: Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2193,7 +2193,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1, diff --git ql/src/test/results/clientpositive/llap/join_emit_interval.q.out ql/src/test/results/clientpositive/llap/join_emit_interval.q.out index 7fda074..eace277 100644 --- ql/src/test/results/clientpositive/llap/join_emit_interval.q.out +++ ql/src/test/results/clientpositive/llap/join_emit_interval.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -75,24 +75,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -113,6 +96,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -141,7 +141,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -161,10 +161,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -179,22 +179,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -215,6 +200,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -222,7 +222,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) diff --git ql/src/test/results/clientpositive/llap/join_filters.q.out ql/src/test/results/clientpositive/llap/join_filters.q.out index b730d30..f0ab223 100644 --- ql/src/test/results/clientpositive/llap/join_filters.q.out +++ ql/src/test/results/clientpositive/llap/join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -25,7 +25,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -39,7 +39,7 @@ POSTHOOK: Input: default@myinput1 48 NULL NULL NULL NULL 40 NULL NULL M3MWtBJdRXSWIJY5Qr/otw== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -53,7 +53,7 @@ NULL NULL 12 35 NULL NULL 48 NULL NULL NULL NULL 40 LNZKrcVNAvaeDALnsg72bw== -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -356,7 +356,7 @@ NULL NULL NULL NULL 12 35 NULL NULL NULL NULL 48 NULL NULL NULL NULL NULL NULL 40 AzUxen/yR7DlsL00zfSITA== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -407,7 +407,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -770,7 +770,7 @@ NULL NULL 48 NULL NULL NULL NULL 135 NULL NULL NULL 35 UBr9lyqgsjDFvooMgQlZ9w== -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -781,7 +781,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -795,7 +795,7 @@ POSTHOOK: Input: default@myinput1 48 NULL NULL NULL NULL 40 NULL NULL M3MWtBJdRXSWIJY5Qr/otw== -Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -809,7 +809,7 @@ NULL NULL 12 35 NULL NULL 48 NULL NULL NULL NULL 40 LNZKrcVNAvaeDALnsg72bw== -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -1112,7 +1112,7 @@ NULL NULL NULL NULL 12 35 NULL NULL NULL NULL 48 NULL NULL NULL NULL NULL NULL 40 AzUxen/yR7DlsL00zfSITA== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -1163,7 +1163,7 @@ POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 100 100 100 100 YaI1msgLVpfEx943Tfea/Q== -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/join_nulls.q.out ql/src/test/results/clientpositive/llap/join_nulls.q.out index c9876d3..3e11fe4 100644 --- ql/src/test/results/clientpositive/llap/join_nulls.q.out +++ ql/src/test/results/clientpositive/llap/join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -32,7 +32,7 @@ POSTHOOK: Input: default@myinput1 NULL 35 100 100 NULL 35 48 NULL NULL 35 NULL 35 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -50,7 +50,7 @@ POSTHOOK: Input: default@myinput1 NULL 35 100 100 NULL 35 48 NULL NULL 35 NULL 35 -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -277,7 +277,7 @@ POSTHOOK: Input: default@myinput1 100 100 100 100 100 100 NULL 35 NULL 35 NULL 35 NULL NULL NULL NULL 48 NULL -Warning: Shuffle Join MERGEJOIN[7][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[7][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/join_nullsafe.q.out ql/src/test/results/clientpositive/llap/join_nullsafe.q.out index 71685aa..b1ddb6b 100644 --- ql/src/test/results/clientpositive/llap/join_nullsafe.q.out +++ ql/src/test/results/clientpositive/llap/join_nullsafe.q.out @@ -27,10 +27,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -43,20 +43,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int) - sort order: + - Map-reduce partition columns: value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -79,6 +66,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int) + sort order: + + Map-reduce partition columns: value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -118,10 +118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -137,7 +137,31 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 key (type: int) + 1 value (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -153,7 +177,7 @@ STAGE PLANS: value expressions: key (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -169,30 +193,6 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 key (type: int) - 1 value (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -223,10 +223,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -239,33 +239,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int) - sort order: + - Map-reduce partition columns: value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -290,6 +264,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int) + sort order: + + Map-reduce partition columns: value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -347,10 +347,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -365,7 +365,32 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 key (type: int), value (type: int) + 1 value (type: int), key (type: int) + 2 key (type: int), value (type: int) + nullSafes: [true, false] + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -380,7 +405,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -395,31 +420,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 key (type: int), value (type: int) - 1 value (type: int), key (type: int) - 2 key (type: int), value (type: int) - nullSafes: [true, false] - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 57 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -450,10 +450,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -465,31 +465,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: int), key (type: int) - sort order: ++ - Map-reduce partition columns: value (type: int), key (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int), value (type: int) - sort order: ++ - Map-reduce partition columns: key (type: int), value (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -514,6 +490,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: int), key (type: int) + sort order: ++ + Map-reduce partition columns: value (type: int), key (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int), value (type: int) + sort order: ++ + Map-reduce partition columns: key (type: int), value (type: int) + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1589,10 +1589,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1608,23 +1608,7 @@ STAGE PLANS: value expressions: value (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is null (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: null (type: int) - sort order: + - Map-reduce partition columns: null (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1647,6 +1631,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: null (type: int) + sort order: + + Map-reduce partition columns: null (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/lateral_view.q.out ql/src/test/results/clientpositive/llap/lateral_view.q.out index 5bde889..5d5c755 100644 --- ql/src/test/results/clientpositive/llap/lateral_view.q.out +++ ql/src/test/results/clientpositive/llap/lateral_view.q.out @@ -28,11 +28,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -78,7 +78,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -94,7 +94,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -622,10 +622,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tmp_pyang_src_rcfile @@ -646,7 +646,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 184 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out index 61b5c12..7441a9c 100644 --- ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out +++ ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out @@ -19,10 +19,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -39,24 +39,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -77,6 +60,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -120,10 +120,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -140,24 +140,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -178,6 +161,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -221,11 +221,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -244,24 +244,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -277,7 +260,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -298,6 +281,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -347,12 +347,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -371,41 +371,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -421,7 +387,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -438,7 +404,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -463,6 +429,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -518,13 +518,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 7 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -541,43 +541,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -598,7 +562,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -614,7 +597,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -633,7 +616,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -649,6 +632,23 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -706,14 +706,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -730,40 +730,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -780,7 +747,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -797,7 +764,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -818,7 +801,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -834,7 +817,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -949,10 +949,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -969,24 +969,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1008,6 +991,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1043,38 +1043,21 @@ on src1.key = src2.key limit 1 offset 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: src2 + alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -1088,7 +1071,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1110,6 +1093,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1153,11 +1153,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1177,24 +1177,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1211,7 +1194,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1233,6 +1216,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1281,12 +1281,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src2 @@ -1306,41 +1306,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1357,7 +1323,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1374,7 +1340,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1400,6 +1366,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1454,13 +1454,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 7 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1477,44 +1477,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Offset of rows: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1536,7 +1499,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Offset of rows: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1553,7 +1536,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1573,7 +1556,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1590,6 +1573,23 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + Vertex 7 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1646,14 +1646,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -1670,40 +1670,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src3 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1720,7 +1687,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1738,7 +1705,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src3 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1759,7 +1742,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1776,7 +1759,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/limit_pushdown.q.out ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index e9cf56d..c1a8f4d 100644 --- ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -100,10 +100,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -120,7 +120,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -187,10 +187,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -214,7 +214,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -283,10 +283,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -310,7 +310,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -379,10 +379,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -404,7 +404,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -472,10 +472,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -496,7 +496,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -570,10 +570,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -594,7 +594,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -668,10 +668,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -694,7 +694,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -787,11 +787,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -814,7 +814,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -829,7 +829,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -904,13 +904,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -934,31 +934,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.3 - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -979,7 +955,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1000,7 +976,31 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1017,7 +1017,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1058,10 +1058,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1079,7 +1079,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1148,10 +1148,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1168,7 +1168,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1315,11 +1315,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1336,7 +1336,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1354,7 +1354,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out index 735ba67..cf7ece0 100644 --- ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out +++ ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -100,10 +100,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -120,7 +120,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -187,11 +187,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -215,7 +215,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -230,7 +230,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -297,11 +297,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -325,7 +325,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -340,7 +340,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -407,11 +407,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -433,7 +433,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -446,7 +446,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 4265 Data size: 25480 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -513,11 +513,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -538,7 +538,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -558,7 +558,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1048 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -625,11 +625,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -650,7 +650,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -670,7 +670,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1048 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -737,11 +737,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -764,7 +764,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -779,7 +779,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 1808 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -870,11 +870,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -897,7 +897,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -912,7 +912,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -979,11 +979,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1001,7 +1001,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1016,7 +1016,7 @@ STAGE PLANS: Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1083,10 +1083,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1103,7 +1103,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1250,11 +1250,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1271,7 +1271,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1289,7 +1289,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/lineage3.q.out ql/src/test/results/clientpositive/llap/lineage3.q.out index f092967..430d369 100644 --- ql/src/test/results/clientpositive/llap/lineage3.q.out +++ ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -180,7 +180,7 @@ PREHOOK: Input: default@src1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"8bf193b0658183be94e2428a79d91d10","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select key, value from src1 where key not in (select key+18 from src1) order by key PREHOOK: type: QUERY @@ -370,7 +370,7 @@ PREHOOK: query: create table src_dp1 (f string, w string, m int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_dp1 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: from src_dp, src_dp1 insert into dest_dp1 partition (year) select first, word, year insert into dest_dp2 partition (y, m) select first, word, year, month diff --git ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index cdb688b..3f4ba09 100644 --- ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/llap_nullscan.q.out ql/src/test/results/clientpositive/llap/llap_nullscan.q.out index b798e82..2c8e34c 100644 --- ql/src/test/results/clientpositive/llap/llap_nullscan.q.out +++ ql/src/test/results/clientpositive/llap/llap_nullscan.q.out @@ -39,7 +39,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -157,11 +157,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -180,25 +180,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src_orc - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -216,7 +198,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -261,11 +261,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -284,7 +286,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src_orc @@ -304,22 +321,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -334,8 +336,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -368,10 +368,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -387,23 +387,7 @@ STAGE PLANS: value expressions: key (type: string), ds (type: string), hr (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string), ds (type: string), hr (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -425,6 +409,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 10 Data size: 3560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), ds (type: string), hr (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/llap_partitioned.q.out ql/src/test/results/clientpositive/llap/llap_partitioned.q.out index d35501e..2afd8fe 100644 --- ql/src/test/results/clientpositive/llap/llap_partitioned.q.out +++ ql/src/test/results/clientpositive/llap/llap_partitioned.q.out @@ -1623,10 +1623,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: oft @@ -1639,7 +1639,7 @@ STAGE PLANS: 1 ctinyint (type: tinyint) outputColumnNames: _col1, _col6, _col7, _col10 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 960 Data size: 497280 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col10 (type: tinyint), _col1 (type: int), _col6 (type: char(255)), _col7 (type: varchar(255)) @@ -1654,7 +1654,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: od @@ -1678,7 +1678,7 @@ STAGE PLANS: Target Input: oft Partition key expr: ctinyint Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Execution mode: vectorized, llap LLAP IO: all inputs diff --git ql/src/test/results/clientpositive/llap/llap_stats.q.out ql/src/test/results/clientpositive/llap/llap_stats.q.out index f6921f1..b8fc80e 100644 --- ql/src/test/results/clientpositive/llap/llap_stats.q.out +++ ql/src/test/results/clientpositive/llap/llap_stats.q.out @@ -100,10 +100,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: llap_stats @@ -126,7 +126,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/llap_udf.q.out ql/src/test/results/clientpositive/llap/llap_udf.q.out index 7a95c8b..acf7038 100644 --- ql/src/test/results/clientpositive/llap/llap_udf.q.out +++ ql/src/test/results/clientpositive/llap/llap_udf.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -95,7 +95,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -135,7 +135,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -181,7 +181,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -229,7 +229,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -267,7 +267,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc diff --git ql/src/test/results/clientpositive/llap/llapdecider.q.out ql/src/test/results/clientpositive/llap/llapdecider.q.out index d514f42..b25e5e2 100644 --- ql/src/test/results/clientpositive/llap/llapdecider.q.out +++ ql/src/test/results/clientpositive/llap/llapdecider.q.out @@ -11,11 +11,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -36,7 +36,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -49,7 +49,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -94,11 +94,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -119,7 +119,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -132,7 +132,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -165,11 +165,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -192,7 +192,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -206,7 +206,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -248,11 +248,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -275,7 +275,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -289,7 +289,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -323,11 +323,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -347,25 +347,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -380,7 +362,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -393,6 +375,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -413,11 +413,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -437,27 +437,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -473,7 +453,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -487,6 +467,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -507,11 +507,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -531,25 +531,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -565,7 +547,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -579,6 +561,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -599,11 +599,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -621,25 +621,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -654,7 +636,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -667,6 +649,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -687,11 +687,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -709,25 +709,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -742,7 +724,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -755,6 +737,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -775,11 +775,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -799,27 +799,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -834,7 +814,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -847,6 +827,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -867,11 +867,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -889,25 +889,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reducer 2 + Vertex 2 Reduce Operator Tree: Merge Join Operator condition map: @@ -922,7 +904,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey0 (type: string) @@ -935,6 +917,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Stage: Stage-0 Fetch Operator @@ -955,11 +955,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -979,27 +979,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1015,7 +995,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1029,6 +1009,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1055,10 +1055,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1081,7 +1081,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1116,10 +1116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1140,7 +1140,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -1174,10 +1174,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1198,7 +1198,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) @@ -1232,10 +1232,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_orc @@ -1258,7 +1258,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out index 6ad9af8..7fdf45a 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out index ce5517a..14e5064 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out @@ -45,10 +45,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -64,7 +64,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out index b34975f..dc7245d 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out @@ -51,7 +51,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out index 9c4e8a8..13af9d1 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out @@ -38,7 +38,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out index c7c4c70..6856c0a 100644 --- ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/lvj_mapjoin.q.out @@ -107,10 +107,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: expod1 @@ -139,7 +139,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 85 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = _col2) (type: boolean) @@ -177,7 +177,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 85 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = _col2) (type: boolean) @@ -195,7 +195,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: expod2 diff --git ql/src/test/results/clientpositive/llap/mapjoin2.q.out ql/src/test/results/clientpositive/llap/mapjoin2.q.out index cdee70b..3a74b84 100644 --- ql/src/test/results/clientpositive/llap/mapjoin2.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin2.q.out @@ -22,7 +22,7 @@ POSTHOOK: type: QUERY POSTHOOK: Output: default@tbl POSTHOOK: Lineage: tbl.n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: tbl.t SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select a.n, a.t, isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a left outer join (select * from tbl where 1 = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -32,7 +32,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### 1 one true true -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), b.n, b.t from (select * from tbl where 2 = 1) a right outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -42,7 +42,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### true true 2 two -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a full outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl diff --git ql/src/test/results/clientpositive/llap/mapjoin3.q.out ql/src/test/results/clientpositive/llap/mapjoin3.q.out index cf1bebb..1aa212e 100644 --- ql/src/test/results/clientpositive/llap/mapjoin3.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin3.q.out @@ -98,10 +98,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -118,7 +118,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col2 (type: varchar(100)), _col3 (type: varchar(100)) @@ -133,7 +133,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test_1 diff --git ql/src/test/results/clientpositive/llap/mapjoin46.q.out ql/src/test/results/clientpositive/llap/mapjoin46.q.out index 73960ce..cc9893c 100644 --- ql/src/test/results/clientpositive/llap/mapjoin46.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin46.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -75,7 +75,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -86,7 +86,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -155,10 +155,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -178,7 +178,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -189,7 +189,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -240,7 +240,7 @@ NULL NULL None NULL NULL NULL 99 2 Mat NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -262,10 +262,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -285,7 +285,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 116 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -296,7 +296,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -321,7 +321,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102 @@ -363,10 +363,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -383,7 +383,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -400,7 +400,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -438,7 +438,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -458,10 +458,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -481,7 +481,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -492,7 +492,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -514,7 +514,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) @@ -541,7 +541,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -565,10 +565,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -585,7 +585,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -597,7 +597,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -619,7 +619,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -651,7 +651,7 @@ NULL NULL None 102 2 Del 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -673,10 +673,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -693,7 +693,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -705,7 +705,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -727,7 +727,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -757,7 +757,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 105 NULL None 101 2 Car 104 3 Fli 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -779,10 +779,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -799,7 +799,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -811,7 +811,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -833,7 +833,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.value=test2.value @@ -881,10 +881,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -901,7 +901,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -913,7 +913,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -962,7 +962,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -986,10 +986,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1004,7 +1004,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1021,7 +1021,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1040,7 +1040,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1072,7 +1072,7 @@ NULL NULL None 102 2 Del 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1094,10 +1094,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1112,7 +1112,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1129,7 +1129,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col0 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1148,7 +1148,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1175,7 +1175,7 @@ POSTHOOK: Input: default@test2 100 1 Bob 104 3 Fli 101 2 Car 105 NULL None 100 1 Bob 105 NULL None -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 RIGHT OUTER JOIN test2 @@ -1197,10 +1197,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1215,7 +1215,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1232,7 +1232,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {((_col1 = _col4) or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1251,7 +1251,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 RIGHT OUTER JOIN test2 ON (test1.value=test2.value @@ -1301,10 +1301,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1321,7 +1321,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1338,7 +1338,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1380,7 +1380,7 @@ POSTHOOK: Input: default@test2 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1404,10 +1404,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1422,22 +1422,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1456,6 +1441,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1463,7 +1463,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1495,7 +1495,7 @@ NULL NULL None 102 2 Del 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1517,10 +1517,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1535,22 +1535,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1569,6 +1554,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1576,7 +1576,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1606,7 +1606,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 103 2 Ema 101 2 Car 104 3 Fli 101 2 Car 105 NULL None -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 FULL OUTER JOIN test2 @@ -1628,10 +1628,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1646,22 +1646,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1680,6 +1665,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1687,7 +1687,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT * FROM test1 FULL OUTER JOIN test2 ON (test1.value=test2.value @@ -1737,10 +1737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1757,24 +1757,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: test2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1793,6 +1776,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: test2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 38 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1827,7 +1827,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -1877,12 +1877,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Map 4 <- Map 5 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1899,7 +1899,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -1916,7 +1916,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 0 Map 1 + 0 Vertex 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1925,7 +1925,26 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} + Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: test1 @@ -1942,7 +1961,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 5 + 1 Vertex 5 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col3 BETWEEN 100 AND 102)} Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1951,7 +1970,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: test2 @@ -1968,25 +1987,6 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 36 Data size: 768 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1994,7 +1994,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1, diff --git ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out index 4539684..adcd96a 100644 --- ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin_decimal.q.out @@ -87,11 +87,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -111,7 +111,7 @@ STAGE PLANS: 1 _col0 (type: decimal(6,2)) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(4,2)) @@ -120,7 +120,21 @@ STAGE PLANS: value expressions: _col1 (type: decimal(4,0)) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(4,0)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -139,20 +153,6 @@ STAGE PLANS: Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(4,0)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out index 81a8546..2bc5874 100644 --- ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out @@ -55,10 +55,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -78,7 +78,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 61 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -89,7 +89,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -134,7 +134,7 @@ NULL NULL None NULL NULL NULL 100 1 Bob NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test1 LEFT OUTER JOIN test2 @@ -154,10 +154,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -177,7 +177,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 24 Data size: 476 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -188,7 +188,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2 @@ -210,7 +210,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT * FROM test1 LEFT OUTER JOIN test2 ON (test1.key between 100 and 102) diff --git ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out index 7c504f1..b44074c 100644 --- ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -31,25 +31,25 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 2 => 25 + Estimated key counts: Vertex 2 => 25 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Position of Big Table: 0 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 500 + Estimated key counts: Vertex 3 => 500 keys: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Position of Big Table: 0 Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -276,7 +276,7 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src1 @@ -351,7 +351,7 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [src1] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -448,10 +448,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -471,7 +471,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 732 Data size: 7782 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -481,7 +481,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 805 Data size: 8560 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -492,7 +492,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src1 @@ -511,7 +511,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -552,11 +552,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -576,7 +576,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -586,7 +586,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -602,7 +602,27 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src @@ -621,7 +641,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src1 @@ -640,26 +660,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/merge1.q.out ql/src/test/results/clientpositive/llap/merge1.q.out index 4bcb728..35248a4 100644 --- ql/src/test/results/clientpositive/llap/merge1.q.out +++ ql/src/test/results/clientpositive/llap/merge1.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -51,7 +51,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -479,7 +479,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src @@ -545,7 +545,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src diff --git ql/src/test/results/clientpositive/llap/merge2.q.out ql/src/test/results/clientpositive/llap/merge2.q.out index aa0567b..f60e6f9 100644 --- ql/src/test/results/clientpositive/llap/merge2.q.out +++ ql/src/test/results/clientpositive/llap/merge2.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -51,7 +51,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -479,7 +479,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src @@ -545,7 +545,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test_src diff --git ql/src/test/results/clientpositive/llap/mergejoin.q.out ql/src/test/results/clientpositive/llap/mergejoin.q.out index 6114548..52f6940 100644 --- ql/src/test/results/clientpositive/llap/mergejoin.q.out +++ ql/src/test/results/clientpositive/llap/mergejoin.q.out @@ -13,12 +13,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -39,7 +39,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -73,25 +91,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -284,13 +284,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -323,27 +323,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -362,7 +342,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -377,7 +357,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -389,6 +369,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1401,13 +1401,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1436,26 +1436,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key BETWEEN DynamicValue(RS_4_a_key_min) AND DynamicValue(RS_4_a_key_max) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1474,7 +1455,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1489,7 +1470,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1501,6 +1482,25 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key BETWEEN DynamicValue(RS_4_a_key_min) AND DynamicValue(RS_4_a_key_max) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1542,13 +1542,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1567,36 +1567,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=500) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1615,7 +1586,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1630,7 +1601,36 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=500) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1683,11 +1683,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1703,23 +1703,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1738,7 +1722,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1753,6 +1737,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -1790,16 +1790,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 8 (BROADCAST_EDGE) - Map 6 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 8 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 1 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (BROADCAST_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1833,60 +1833,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1902,7 +1849,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 56100 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1921,7 +1868,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1936,7 +1883,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1948,18 +1895,71 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 8 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 6 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25) + mode: final + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Stage: Stage-0 Fetch Operator @@ -1997,13 +1997,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2036,27 +2036,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2075,7 +2055,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2090,7 +2070,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2102,6 +2082,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2145,15 +2145,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 8 <- Union 3 (CONTAINS) - Map 9 <- Reducer 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Union 3 (PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS) + Vertex 9 <- Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2173,80 +2175,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: s3 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: s2 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 9 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2275,7 +2204,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2294,7 +2223,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2309,7 +2238,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2321,8 +2250,79 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Union 3 - Vertex: Union 3 + Vertex 7 + Map Operator Tree: + TableScan + alias: s3 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: s2 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2343,13 +2343,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2382,27 +2382,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2421,7 +2401,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2436,7 +2416,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2448,6 +2428,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: b + filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2483,16 +2483,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 8 (BROADCAST_EDGE) - Map 6 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 8 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 1 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (BROADCAST_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2503,83 +2503,30 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=242) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - filterExpr: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Select Operator - expressions: _col0 (type: string) + expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=242) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2595,7 +2542,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 550 Data size: 56100 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2614,7 +2561,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2629,7 +2576,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2641,7 +2588,60 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 8 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2698,15 +2698,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 8 <- Union 3 (CONTAINS) - Map 9 <- Reducer 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 9 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Union 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Union 3 (PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS) + Vertex 9 <- Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2726,80 +2728,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: s3 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 8 - Map Operator Tree: - TableScan - alias: s2 - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 9 - Map Operator Tree: - TableScan - alias: b - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2828,7 +2757,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2847,7 +2776,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2862,7 +2791,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2874,8 +2803,79 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Union 3 - Vertex: Union 3 + Vertex 7 + Map Operator Tree: + TableScan + alias: s3 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: s2 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 508 Data size: 51836 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=508) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: b + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -2910,15 +2910,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 2 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -2937,26 +2937,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 24684 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 6 - Map Operator Tree: - TableScan - alias: t2 - filterExpr: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -2981,7 +2962,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3000,7 +2981,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3015,7 +2996,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3027,7 +3008,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t2 + filterExpr: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean) + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 500 Data size: 51000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -3073,7 +3073,7 @@ POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### 480 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -3093,7 +3093,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 0 val_0 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a right outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -3108,7 +3108,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -3129,8 +3129,8 @@ POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -3153,8 +3153,8 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join @@ -3681,8 +3681,8 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join @@ -4203,7 +4203,7 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[30][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -4224,7 +4224,7 @@ POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[17][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[17][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join diff --git ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out index 25be543..2f9e209 100644 --- ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out +++ ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out @@ -191,10 +191,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl @@ -214,7 +214,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -251,10 +251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -274,7 +274,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -311,10 +311,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl @@ -334,7 +334,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -398,7 +398,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: decimal(11,1)), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: int), _col9 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -625,10 +625,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_tbl_part @@ -648,7 +648,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/metadataonly1.q.out ql/src/test/results/clientpositive/llap/metadataonly1.q.out index 27218cf..93c94ad 100644 --- ql/src/test/results/clientpositive/llap/metadataonly1.q.out +++ ql/src/test/results/clientpositive/llap/metadataonly1.q.out @@ -19,10 +19,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -46,7 +46,7 @@ STAGE PLANS: auto parallelism: false Execution mode: llap LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -113,10 +113,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -191,7 +191,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -332,7 +332,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -394,10 +394,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -473,7 +473,7 @@ STAGE PLANS: name: default.test1 Truncated Path -> Alias: /test1/ds=1 [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -544,12 +544,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a2 @@ -669,7 +669,62 @@ STAGE PLANS: Truncated Path -> Alias: /test1/ds=1 [a2] /test1/ds=2 [a2] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Position of Big Table: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: test1 @@ -793,62 +848,7 @@ STAGE PLANS: Truncated Path -> Alias: /test1/ds=1 [test1] /test1/ds=2 [test1] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col0 (type: bigint) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 5 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -883,7 +883,7 @@ STAGE PLANS: Target Input: a2 Partition key expr: ds Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Target Vertex: Map 1 + Target Vertex: Vertex 1 Stage: Stage-0 Fetch Operator @@ -946,10 +946,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1125,7 +1125,7 @@ STAGE PLANS: nullscan://null/default.test2/part_ds=1_hr=1_ [test2] nullscan://null/default.test2/part_ds=1_hr=2_ [test2] nullscan://null/default.test2/part_ds=1_hr=3_ [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1191,10 +1191,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1372,7 +1372,7 @@ STAGE PLANS: /test2/ds=1/hr=1 [test2] /test2/ds=1/hr=2 [test2] /test2/ds=1/hr=3 [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1438,10 +1438,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test1 @@ -1564,7 +1564,7 @@ STAGE PLANS: Truncated Path -> Alias: nullscan://null/default.test1/part_ds=1_ [test1] nullscan://null/default.test1/part_ds=2_ [test1] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1677,10 +1677,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2 @@ -1954,7 +1954,7 @@ STAGE PLANS: nullscan://null/default.test2/part_ds=1_hr=1_ [test2] nullscan://null/default.test2/part_ds=1_hr=2_ [test2] nullscan://null/default.test2/part_ds=1_hr=3_ [test2] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/mrr.q.out ql/src/test/results/clientpositive/llap/mrr.q.out index 1437d5d..2118af1 100644 --- ql/src/test/results/clientpositive/llap/mrr.q.out +++ ql/src/test/results/clientpositive/llap/mrr.q.out @@ -11,11 +11,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -38,7 +38,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -52,7 +52,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -403,12 +403,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -427,27 +427,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -468,7 +448,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 609 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -486,7 +466,7 @@ STAGE PLANS: key expressions: _col1 (type: bigint), _col0 (type: string) sort order: ++ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -500,6 +480,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -837,12 +837,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -861,7 +861,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: s2 @@ -881,7 +881,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1, _col2 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col1 (type: string), _col2 (type: string) @@ -895,7 +895,7 @@ STAGE PLANS: Statistics: Num rows: 609 Data size: 108402 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -913,7 +913,7 @@ STAGE PLANS: key expressions: _col1 (type: bigint), _col0 (type: string) sort order: ++ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1292,61 +1292,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1368,7 +1324,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1382,7 +1338,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1395,7 +1351,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1409,7 +1365,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1422,7 +1378,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1444,7 +1400,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 68 Data size: 19380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1458,7 +1414,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1475,7 +1453,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 68 Data size: 6460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1488,6 +1466,28 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 68 Data size: 6460 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1686,11 +1686,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1712,27 +1712,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1749,14 +1729,14 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1770,6 +1750,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out index 73ccdf5..a040628 100644 --- ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out +++ ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out @@ -184,11 +184,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -208,7 +208,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -221,7 +221,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -234,7 +234,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -253,7 +268,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -272,21 +287,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -347,11 +347,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -371,7 +371,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -384,7 +384,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -397,7 +397,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -416,7 +431,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -435,21 +450,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -512,11 +512,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -536,7 +536,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 58432 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) @@ -550,7 +550,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 64275 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -566,7 +566,27 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -585,7 +605,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -605,26 +625,6 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3025 Data size: 32137 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -817,11 +817,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -841,7 +841,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -855,7 +855,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -869,7 +869,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -883,7 +883,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -900,7 +900,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -919,7 +934,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -939,7 +954,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -958,7 +973,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -977,21 +992,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1152,11 +1152,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1176,7 +1176,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -1190,7 +1190,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -1204,7 +1204,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -1218,7 +1218,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -1235,7 +1235,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -1254,7 +1269,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -1274,7 +1289,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -1293,7 +1308,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -1312,21 +1327,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1487,11 +1487,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 5 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1511,7 +1511,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col2 (type: string) @@ -1525,7 +1525,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: string), _col3 (type: string), _col4 (type: string) @@ -1539,7 +1539,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col4 (type: string), _col5 (type: string) @@ -1553,7 +1553,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 7320 Data size: 105677 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0) (type: int), hash(_col1) (type: int), hash(_col2) (type: int), hash(_col3) (type: int), hash(_col4) (type: int), hash(_col7) (type: int), hash(_col5) (type: int), hash(_col6) (type: int) @@ -1570,7 +1570,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: smalltbl1 @@ -1589,7 +1604,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: smalltbl2 @@ -1609,7 +1624,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: smalltbl3 @@ -1628,7 +1643,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: smalltbl4 @@ -1647,21 +1662,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1822,14 +1822,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: bigtbl @@ -1849,7 +1849,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: smalltbl4 @@ -1868,65 +1868,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: smalltbl3 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1947,7 +1889,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 5500 Data size: 79398 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1968,7 +1910,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 6050 Data size: 87337 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1989,7 +1931,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 6655 Data size: 96070 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2013,7 +1955,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2028,6 +1970,64 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 7 + Map Operator Tree: + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: smalltbl3 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out index 433c701..c9dfdf9 100644 --- ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out +++ ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out @@ -21,12 +21,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -46,7 +48,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -54,7 +56,21 @@ STAGE PLANS: Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -73,7 +89,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: x2 @@ -93,7 +109,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -101,7 +117,7 @@ STAGE PLANS: Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -120,22 +136,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -259,12 +259,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -283,7 +285,36 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y1 @@ -302,7 +333,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x2 @@ -321,7 +352,22 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y2 @@ -340,52 +386,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 120 Data size: 10440 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -509,12 +509,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Map 6 (BROADCAST_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 6 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: x1 @@ -535,7 +537,33 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: x2 @@ -555,7 +583,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) @@ -563,7 +591,7 @@ STAGE PLANS: Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -582,34 +610,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 72 Data size: 6252 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 72 Data size: 6264 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -722,15 +722,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -750,7 +750,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -764,7 +764,64 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Vertex 7 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y2 @@ -783,7 +840,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x1 @@ -803,7 +860,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -817,7 +874,20 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y1 @@ -836,76 +906,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Reducer 7 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -990,15 +990,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1018,7 +1018,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1032,7 +1032,64 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Vertex 7 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y2 @@ -1051,7 +1108,7 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: x1 @@ -1071,7 +1128,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1085,7 +1142,20 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 8 Map Operator Tree: TableScan alias: y1 @@ -1104,76 +1174,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Reducer 7 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -1258,14 +1258,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1285,7 +1285,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1299,46 +1299,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1354,7 +1315,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Reducer 7 + 1 Vertex 7 Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1368,7 +1329,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1381,7 +1342,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1395,7 +1356,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1492,14 +1492,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (BROADCAST_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 1 <- Vertex 5 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x2 @@ -1519,7 +1519,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1533,46 +1533,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: y2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1588,7 +1549,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col1 input vertices: - 1 Reducer 7 + 1 Vertex 7 Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1602,7 +1563,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1615,7 +1576,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1629,7 +1590,46 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: y2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1751,11 +1751,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -1774,7 +1774,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 137 Data size: 2487 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1787,7 +1787,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: y @@ -1806,21 +1821,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1877,13 +1877,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Map 6 <- Map 8 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (BROADCAST_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (BROADCAST_EDGE), Union 3 (CONTAINS) + Vertex 1 <- Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (BROADCAST_EDGE) + Vertex 6 <- Vertex 8 (BROADCAST_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1899,7 +1901,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1913,7 +1915,36 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) + outputColumnNames: _col1 + input vertices: + 1 Vertex 5 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1928,7 +1959,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1943,7 +1974,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: a @@ -1959,7 +1990,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) @@ -1973,37 +2004,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2019,7 +2020,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1 input vertices: - 1 Map 5 + 1 Vertex 9 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string) @@ -2032,37 +2033,36 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col1 - input vertices: - 1 Map 9 + LLAP IO: no inputs + Vertex 9 + Map Operator Tree: + TableScan + alias: c Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out index 0cbb9de..06ade11 100644 --- ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out +++ ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out @@ -32,11 +32,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: employee @@ -57,7 +57,7 @@ STAGE PLANS: Statistics: Num rows: 36 Data size: 198 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -78,7 +78,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -173,11 +173,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: employee @@ -198,7 +198,7 @@ STAGE PLANS: Statistics: Num rows: 60 Data size: 330 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -219,7 +219,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/multi_insert.q.out ql/src/test/results/clientpositive/llap/multi_insert.q.out index 58fc759..b40c90f 100644 --- ql/src/test/results/clientpositive/llap/multi_insert.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -180,7 +180,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -323,7 +323,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -466,7 +466,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -609,10 +609,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -627,7 +627,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -763,10 +763,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -781,7 +781,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -917,10 +917,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -935,7 +935,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -1071,10 +1071,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1089,7 +1089,7 @@ STAGE PLANS: Statistics: Num rows: 221 Data size: 39338 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Forward @@ -1225,11 +1225,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1262,7 +1264,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1295,8 +1297,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1422,11 +1422,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1459,7 +1461,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1492,8 +1494,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1619,11 +1619,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1656,7 +1658,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1689,8 +1691,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -1816,11 +1816,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1853,7 +1855,7 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1886,8 +1888,6 @@ STAGE PLANS: name: default.src_multi2 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-3 Dependency Collection @@ -2011,7 +2011,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2114,7 +2114,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2217,7 +2217,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2320,7 +2320,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out index e8a3610..25650cd 100644 --- ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out @@ -57,7 +57,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -277,11 +277,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -374,7 +374,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -391,7 +391,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -519,11 +519,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -582,7 +582,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -599,7 +599,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Forward @@ -776,12 +776,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -901,7 +901,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -918,7 +918,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -935,7 +935,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1132,12 +1132,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src_10 @@ -1234,7 +1234,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 62 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1251,7 +1251,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv1 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1268,7 +1268,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_lv2 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Forward diff --git ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out index 1441ed0..049cc7c 100644 --- ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out +++ ql/src/test/results/clientpositive/llap/nonmr_fetch_threshold.q.out @@ -118,12 +118,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=500 width=184) - Output:["_col0"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 PREHOOK: query: explain select key, value from src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/offset_limit.q.out ql/src/test/results/clientpositive/llap/offset_limit.q.out index adfeb05..b6b29be 100644 --- ql/src/test/results/clientpositive/llap/offset_limit.q.out +++ ql/src/test/results/clientpositive/llap/offset_limit.q.out @@ -13,11 +13,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -56,7 +56,7 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index fb8e3ad..364e4e2 100644 --- ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -33,7 +33,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -101,10 +101,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -121,7 +121,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -189,10 +189,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -216,7 +216,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -286,10 +286,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -313,7 +313,7 @@ STAGE PLANS: value expressions: _col1 (type: struct) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -383,10 +383,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -408,7 +408,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -501,7 +501,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -576,10 +576,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -600,7 +600,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -675,10 +675,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -701,7 +701,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -795,11 +795,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -822,7 +822,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -836,7 +836,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -904,10 +904,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -925,7 +925,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -995,10 +995,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1015,7 +1015,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1133,11 +1133,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1154,7 +1154,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1172,7 +1172,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1298,13 +1298,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1321,24 +1321,7 @@ STAGE PLANS: TopN Hash Memory Usage: 2.0E-5 Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 2.0E-5 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1360,7 +1343,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1382,7 +1365,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 2.0E-5 + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1400,7 +1400,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1900 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 value expressions: _col0 (type: string), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out index 4c08ee1..56bf1d4 100644 --- ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out +++ ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out @@ -46,10 +46,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart @@ -76,7 +76,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -142,11 +142,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -220,7 +220,42 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -441,42 +476,7 @@ STAGE PLANS: nullscan://null/default.srcpart/part_ds=2008-04-08_hr=12_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=11_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=12_ [srcpart] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -535,11 +535,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -613,7 +615,38 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: srcpart @@ -837,7 +870,7 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] - Reducer 2 + Vertex 5 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -868,39 +901,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Reducer 5 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -928,7 +928,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 0 2000 -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain extended select * from (select key from src where false) a left outer join (select value from srcpart limit 0) b PREHOOK: type: QUERY @@ -944,11 +944,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1021,7 +1021,42 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: srcpart @@ -1242,42 +1277,7 @@ STAGE PLANS: nullscan://null/default.srcpart/part_ds=2008-04-08_hr=12_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=11_ [srcpart] nullscan://null/default.srcpart/part_ds=2008-04-09_hr=12_ [srcpart] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1302,7 +1302,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [a, b]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from (select key from src where false) a left outer join (select value from srcpart limit 0) b PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -1336,11 +1336,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1429,7 +1431,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1499,27 +1501,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Map 5 - Map Operator Tree: - TableScan - alias: srcpart - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -1554,8 +1536,26 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator @@ -1588,10 +1588,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -1662,7 +1662,46 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [s1] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -1733,45 +1772,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [s2] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 value (type: string) - 1 value (type: string) - outputColumnNames: _col0, _col1, _col5, _col6 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1802,10 +1802,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1879,7 +1879,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: nullscan://null/default.src/part_ [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/orc_llap.q.out ql/src/test/results/clientpositive/llap/orc_llap.q.out index 4fb3d12..246cb62 100644 --- ql/src/test/results/clientpositive/llap/orc_llap.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap.q.out @@ -81,7 +81,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: default@cross_numbers POSTHOOK: Lineage: cross_numbers.i EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers @@ -121,7 +121,7 @@ POSTHOOK: Output: default@orc_llap_small POSTHOOK: Lineage: orc_llap_small.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: orc_llap_small.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_llap_small.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select count(1) from orc_llap_small y join orc_llap_small x PREHOOK: type: QUERY @@ -137,11 +137,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: y @@ -155,7 +155,7 @@ STAGE PLANS: 0 1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 225 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) @@ -168,19 +168,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 15 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -195,6 +183,18 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 15 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -202,7 +202,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select count(1) from orc_llap_small y join orc_llap_small x PREHOOK: type: QUERY PREHOOK: Input: default@orc_llap_small @@ -245,10 +245,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -272,7 +272,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -318,10 +318,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -345,7 +345,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -391,10 +391,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -418,7 +418,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -464,11 +464,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -491,7 +491,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -513,7 +513,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -559,13 +559,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: o1 @@ -599,28 +599,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: o2 - filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: csmallint (type: smallint), cstring2 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: smallint) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -644,7 +623,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -659,7 +638,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -671,6 +650,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: o2 + filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: csmallint (type: smallint), cstring2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: smallint) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -687,7 +687,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_llap #### A masked pattern was here #### -735462183586256 -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers @@ -737,10 +737,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -764,7 +764,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -810,10 +810,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -837,7 +837,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -883,10 +883,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -910,7 +910,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -956,11 +956,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap @@ -983,7 +983,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1005,7 +1005,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1051,13 +1051,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 4 <- Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: o1 @@ -1091,28 +1091,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: o2 - filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: csmallint (type: smallint), cstring2 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: smallint) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1136,7 +1115,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1151,7 +1130,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1163,6 +1142,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: smallint), _col2 (type: binary) + Vertex 5 + Map Operator Tree: + TableScan + alias: o2 + filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: csmallint (type: smallint), cstring2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: smallint) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index bc3915a..05dae47 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -259,9 +259,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 @@ -296,9 +296,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -323,9 +323,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 22 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 22 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -348,9 +348,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 16 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 16 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -373,9 +373,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -398,9 +398,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -423,9 +423,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -448,9 +448,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -473,9 +473,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1697 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1697 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -498,9 +498,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -523,9 +523,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1713 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1713 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -548,9 +548,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -573,9 +573,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 50 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 50 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -598,9 +598,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 318 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 318 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -651,9 +651,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -671,9 +671,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -705,9 +705,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -733,9 +733,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -758,9 +758,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -797,9 +797,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -822,9 +822,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -842,9 +842,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -867,9 +867,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -892,9 +892,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -917,9 +917,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 81 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 81 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -942,9 +942,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 74 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 74 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -967,9 +967,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -992,9 +992,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 13 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 13 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1017,9 +1017,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1042,9 +1042,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 7 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 7 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1067,9 +1067,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1087,9 +1087,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1107,9 +1107,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1127,9 +1127,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1152,9 +1152,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1177,9 +1177,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1202,9 +1202,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index 5c54622..056c768 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -259,9 +259,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2094 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2094 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -285,9 +285,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2094 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2094 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out index 01adcd4..d5fca30 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out @@ -80,7 +80,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap_nonvector @@ -231,7 +231,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_llap_nonvector diff --git ql/src/test/results/clientpositive/llap/orc_merge5.q.out ql/src/test/results/clientpositive/llap/orc_merge5.q.out index 47096bb..6a49d3e 100644 --- ql/src/test/results/clientpositive/llap/orc_merge5.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge5.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -131,7 +131,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge6.q.out ql/src/test/results/clientpositive/llap/orc_merge6.q.out index 3969a96..b19ceb6 100644 --- ql/src/test/results/clientpositive/llap/orc_merge6.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge6.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -176,7 +176,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge7.q.out ql/src/test/results/clientpositive/llap/orc_merge7.q.out index d86d2dc..6545871 100644 --- ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 @@ -210,7 +210,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out index e5101f4..089db10 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index 6fcb45a..38877b1 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_merge5 diff --git ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index e28ed5d..fa0e882 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -211,9 +211,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 @@ -248,9 +248,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -275,9 +275,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 22 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 22 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -300,9 +300,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 16 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 16 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -325,9 +325,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -350,9 +350,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -375,9 +375,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -400,9 +400,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -425,9 +425,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1697 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1697 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -450,9 +450,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -475,9 +475,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1713 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1713 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -500,9 +500,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -525,9 +525,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 50 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 50 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -550,9 +550,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 318 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 318 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -603,9 +603,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -623,9 +623,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -657,9 +657,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -685,9 +685,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -710,9 +710,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -749,9 +749,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -774,9 +774,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -794,9 +794,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -819,9 +819,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -844,9 +844,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -869,9 +869,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 81 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 81 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -894,9 +894,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 74 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 74 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -919,9 +919,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 12 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -944,9 +944,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 13 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 13 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -969,9 +969,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -994,9 +994,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 7 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 7 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1019,9 +1019,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1039,9 +1039,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1059,9 +1059,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 0 + RECORDS_IN_Vertex_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1079,9 +1079,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1104,9 +1104,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1100 + RECORDS_IN_Vertex_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1129,9 +1129,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1154,9 +1154,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 100 + RECORDS_IN_Vertex_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1179,9 +1179,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 8400 @@ -1207,9 +1207,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -1232,9 +1232,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -1257,9 +1257,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 diff --git ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out index 4cc8984..b91561a 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out @@ -211,9 +211,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -251,9 +251,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -276,9 +276,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -301,9 +301,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -326,9 +326,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -355,9 +355,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -388,9 +388,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -407,9 +407,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -426,9 +426,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -445,9 +445,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -468,9 +468,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -501,9 +501,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -520,9 +520,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -539,9 +539,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -558,9 +558,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -581,9 +581,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 0 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 0 @@ -614,9 +614,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -633,9 +633,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -652,9 +652,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -671,9 +671,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2000 + RECORDS_IN_Vertex_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -694,9 +694,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1566 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1566 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 1566 @@ -713,9 +713,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1566 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 1566 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 1566 @@ -732,9 +732,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -751,9 +751,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 8 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 8 @@ -770,9 +770,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -789,9 +789,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 18 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 18 @@ -808,9 +808,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 8400 @@ -836,9 +836,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4809 CACHE_MISS_BYTES: 0 @@ -865,9 +865,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -884,9 +884,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -907,9 +907,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -926,9 +926,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 2 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 2 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 2 @@ -945,9 +945,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -973,9 +973,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1002,9 +1002,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1021,9 +1021,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1044,9 +1044,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1063,9 +1063,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1086,9 +1086,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1105,9 +1105,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 6 @@ -1128,9 +1128,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1153,9 +1153,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 6 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1182,9 +1182,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1201,9 +1201,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1220,9 +1220,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 1000 + RECORDS_IN_Vertex_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 @@ -1239,9 +1239,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Vertex_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 4 + RECORDS_OUT_INTERMEDIATE_Vertex_1: 4 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 1 4 diff --git ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out index 4bba265..6a70e52 100644 --- ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out @@ -126,10 +126,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -149,7 +149,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -184,10 +184,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -207,7 +207,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -314,10 +314,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -340,7 +340,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -381,10 +381,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -408,7 +408,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -582,10 +582,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -603,7 +603,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 8602 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -647,10 +647,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -669,7 +669,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 8602 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -779,10 +779,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -802,7 +802,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -857,10 +857,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -881,7 +881,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1002,11 +1002,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -1026,7 +1026,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1042,7 +1042,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 296 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1101,11 +1101,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: orc_pred @@ -1126,7 +1126,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1142,7 +1142,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 296 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/parallel.q.out ql/src/test/results/clientpositive/llap/parallel.q.out index 7dba122..6212a95 100644 --- ql/src/test/results/clientpositive/llap/parallel.q.out +++ ql/src/test/results/clientpositive/llap/parallel.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -62,7 +62,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -75,7 +75,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Forward diff --git ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out index 01348a6..f985d57 100644 --- ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out @@ -118,10 +118,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -141,7 +141,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -176,10 +176,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -199,7 +199,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -520,10 +520,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -541,7 +541,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -585,10 +585,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -607,7 +607,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -717,10 +717,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -740,7 +740,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -795,10 +795,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -819,7 +819,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -985,11 +985,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1009,7 +1009,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1025,7 +1025,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1084,11 +1084,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1109,7 +1109,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1125,7 +1125,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1196,11 +1196,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tbl_pred @@ -1221,7 +1221,7 @@ STAGE PLANS: value expressions: _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1237,7 +1237,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: int), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/partition_multilevels.q.out ql/src/test/results/clientpositive/llap/partition_multilevels.q.out index 5082734..23fe671 100644 --- ql/src/test/results/clientpositive/llap/partition_multilevels.q.out +++ ql/src/test/results/clientpositive/llap/partition_multilevels.q.out @@ -975,10 +975,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: partition_test_multilevel @@ -1001,7 +1001,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1565,10 +1565,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: partition_test_multilevel @@ -1591,7 +1591,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/ppd_union_view.q.out ql/src/test/results/clientpositive/llap/ppd_union_view.q.out index 543d176..caaf251 100644 --- ql/src/test/results/clientpositive/llap/ppd_union_view.q.out +++ ql/src/test/results/clientpositive/llap/ppd_union_view.q.out @@ -145,11 +145,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1_new @@ -191,7 +193,7 @@ STAGE PLANS: MultiFileSpray: false Execution mode: llap LLAP IO: unknown - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1_old @@ -270,7 +272,50 @@ STAGE PLANS: name: default.t1_old Truncated Path -> Alias: /t1_old/ds=2011-10-13 [t1_old] - Map 5 + Vertex 4 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col1, _col3 + Position of Big Table: 0 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col3 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), '2011-10-13' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 5 Map Operator Tree: TableScan alias: t1_mapping @@ -349,51 +394,6 @@ STAGE PLANS: name: default.t1_mapping Truncated Path -> Alias: /t1_mapping/ds=2011-10-13 [t1_mapping] - Reducer 4 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col3 - Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2011-10-13' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: PARTIAL -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2 - columns.types string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -454,11 +454,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: t1_new @@ -548,7 +550,7 @@ STAGE PLANS: name: default.t1_new Truncated Path -> Alias: /t1_new/ds=2011-10-15 [t1_new] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: t1_old @@ -575,34 +577,7 @@ STAGE PLANS: auto parallelism: true Execution mode: llap LLAP IO: unknown - Map 5 - Map Operator Tree: - TableScan - alias: t1_mapping - properties: - insideView TRUE - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: key (type: string), keymap (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL - tag: 1 - value expressions: _col0 (type: string) - auto parallelism: true - Execution mode: llap - LLAP IO: unknown - Reducer 4 + Vertex 4 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -645,8 +620,33 @@ STAGE PLANS: TotalFiles: 1 GatherStats: false MultiFileSpray: false - Union 2 - Vertex: Union 2 + Vertex 5 + Map Operator Tree: + TableScan + alias: t1_mapping + properties: + insideView TRUE + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), keymap (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: PARTIAL + tag: 1 + value expressions: _col0 (type: string) + auto parallelism: true + Execution mode: llap + LLAP IO: unknown Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/ptf.q.out ql/src/test/results/clientpositive/llap/ptf.q.out index df55238..009ab91 100644 --- ql/src/test/results/clientpositive/llap/ptf.q.out +++ ql/src/test/results/clientpositive/llap/ptf.q.out @@ -27,11 +27,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -44,7 +44,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -71,7 +71,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -201,12 +201,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -222,22 +222,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p2 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -254,7 +239,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -281,7 +266,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -320,6 +305,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p2 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -392,10 +392,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -408,7 +408,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -516,11 +516,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -533,7 +533,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -560,7 +560,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -696,11 +696,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -713,7 +713,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -740,7 +740,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -879,11 +879,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -896,7 +896,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -931,7 +931,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1064,11 +1064,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1081,22 +1081,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1126,7 +1111,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 23062 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1144,6 +1129,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1218,11 +1218,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -1237,20 +1237,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_name (type: string) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1272,7 +1259,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_name (type: string) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1376,11 +1376,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1408,7 +1408,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1435,7 +1435,7 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1550,11 +1550,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1583,7 +1583,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1611,7 +1611,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1743,11 +1743,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1760,7 +1760,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1787,7 +1787,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1921,12 +1921,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1939,7 +1939,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1989,7 +1989,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2024,7 +2024,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2166,11 +2166,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2183,7 +2183,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2210,7 +2210,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2348,12 +2348,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2366,22 +2366,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2411,7 +2396,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 13078 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2428,7 +2413,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6699 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2493,6 +2478,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2575,11 +2575,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2592,7 +2592,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2627,7 +2627,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2734,11 +2734,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2761,7 +2761,7 @@ STAGE PLANS: value expressions: _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2794,7 +2794,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2964,13 +2964,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2983,7 +2983,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3016,7 +3016,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3069,7 +3069,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.part_4 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3106,7 +3106,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: sum_window_0 (type: bigint), _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3357,12 +3357,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3375,7 +3375,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3432,7 +3432,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3467,7 +3467,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3631,13 +3631,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3650,7 +3650,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3684,7 +3684,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3711,7 +3711,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3738,7 +3738,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3898,12 +3898,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3916,7 +3916,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3950,7 +3950,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3984,7 +3984,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4144,13 +4144,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4163,7 +4163,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4197,7 +4197,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4240,7 +4240,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4268,7 +4268,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4430,12 +4430,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4448,7 +4448,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4498,7 +4498,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4533,7 +4533,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4689,12 +4689,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4707,7 +4707,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4757,7 +4757,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4785,7 +4785,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out index 6c60f98..89d7501 100644 --- ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out +++ ql/src/test/results/clientpositive/llap/ptf_matchpath.q.out @@ -65,10 +65,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -81,7 +81,7 @@ STAGE PLANS: value expressions: origin_city_name (type: string), arr_delay (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -193,10 +193,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -209,7 +209,7 @@ STAGE PLANS: value expressions: origin_city_name (type: string), arr_delay (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -311,10 +311,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: flights_tiny @@ -334,7 +334,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col5 (type: float) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/ptf_streaming.q.out ql/src/test/results/clientpositive/llap/ptf_streaming.q.out index 6013c11..ca18afe 100644 --- ql/src/test/results/clientpositive/llap/ptf_streaming.q.out +++ ql/src/test/results/clientpositive/llap/ptf_streaming.q.out @@ -27,11 +27,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -44,7 +44,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -71,7 +71,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -201,12 +201,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: p1 @@ -222,22 +222,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p2 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -254,7 +239,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -281,7 +266,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -320,6 +305,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p2 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -394,11 +394,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -411,22 +411,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -456,7 +441,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 23062 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -474,6 +459,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -548,11 +548,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -580,7 +580,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -607,7 +607,7 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -722,11 +722,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -755,7 +755,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -783,7 +783,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -917,12 +917,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -935,7 +935,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -985,7 +985,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1020,7 +1020,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1156,12 +1156,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1174,7 +1174,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1224,7 +1224,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1259,7 +1259,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1395,12 +1395,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1413,7 +1413,7 @@ STAGE PLANS: value expressions: p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1463,7 +1463,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1498,7 +1498,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1638,12 +1638,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1656,22 +1656,7 @@ STAGE PLANS: value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_partkey (type: int) - sort order: + - Map-reduce partition columns: p_partkey (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1701,7 +1686,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 13078 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1718,7 +1703,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 29 Data size: 6699 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int), _col7 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1783,6 +1768,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_partkey (type: int) + sort order: + + Map-reduce partition columns: p_partkey (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1887,12 +1887,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1905,7 +1905,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1962,7 +1962,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1997,7 +1997,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2161,13 +2161,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2180,7 +2180,7 @@ STAGE PLANS: value expressions: p_name (type: string), p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2214,7 +2214,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2241,7 +2241,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2268,7 +2268,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2426,12 +2426,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2444,7 +2444,7 @@ STAGE PLANS: value expressions: p_size (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2494,7 +2494,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2522,7 +2522,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out index 2cc6f5d..9e4a252 100644 --- ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out +++ ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -100,7 +100,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -267,10 +267,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: complex_tbl_2 @@ -306,7 +306,7 @@ STAGE PLANS: tag: -1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: bigint), _col5 (type: string), _col6 (type: string) auto parallelism: true - Reducer 2 + Vertex 2 Needs Tagging: false Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out index 08a9bd1..53b1f88 100644 --- ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out +++ ql/src/test/results/clientpositive/llap/reduce_deduplicate_extended.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -30,7 +30,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -70,10 +70,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -90,7 +90,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -134,10 +134,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -154,7 +154,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -198,10 +198,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -224,7 +224,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -260,11 +260,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -284,26 +284,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -326,7 +307,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 30 Data size: 5580 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -346,6 +327,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -366,11 +366,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -390,26 +390,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -424,7 +405,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -438,6 +419,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -458,10 +458,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -482,7 +482,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -526,10 +526,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -550,7 +550,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2570,10 +2570,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2589,7 +2589,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2629,10 +2629,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2649,7 +2649,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2693,10 +2693,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2713,7 +2713,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2757,10 +2757,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2776,7 +2776,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2812,11 +2812,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2836,26 +2836,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2871,7 +2852,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2891,6 +2872,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2911,11 +2911,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2935,26 +2935,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2969,7 +2950,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2983,6 +2964,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3003,10 +3003,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3022,7 +3022,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3066,10 +3066,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3085,7 +3085,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/sample1.q.out ql/src/test/results/clientpositive/llap/sample1.q.out index 61a3a0f..630690e 100644 --- ql/src/test/results/clientpositive/llap/sample1.q.out +++ ql/src/test/results/clientpositive/llap/sample1.q.out @@ -27,7 +27,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s diff --git ql/src/test/results/clientpositive/llap/sample10.q.out ql/src/test/results/clientpositive/llap/sample10.q.out index a3da787..36eacaf 100644 --- ql/src/test/results/clientpositive/llap/sample10.q.out +++ ql/src/test/results/clientpositive/llap/sample10.q.out @@ -48,11 +48,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpartbucket @@ -292,7 +292,7 @@ STAGE PLANS: /srcpartbucket/ds=2008-04-08/hr=12/000000_0 [srcpartbucket] /srcpartbucket/ds=2008-04-09/hr=11/000000_0 [srcpartbucket] /srcpartbucket/ds=2008-04-09/hr=12/000000_0 [srcpartbucket] - Reducer 2 + Vertex 2 Execution mode: llap Needs Tagging: false Reduce Operator Tree: @@ -310,7 +310,7 @@ STAGE PLANS: tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false - Reducer 3 + Vertex 3 Execution mode: llap Needs Tagging: false Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out index a3a54d1..656ae4b 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_acid_part.q.out @@ -526,7 +526,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_multi_ints_string_group @@ -705,7 +705,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_floating_string_group diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out index 81d7a24..7a43540 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_acidvec_part.q.out @@ -526,7 +526,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_multi_ints_string_group @@ -705,7 +705,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_change_numeric_group_string_group_floating_string_group diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out index cc98981..0628640 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part.q.out @@ -66,12 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=101) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_permute_select PREHOOK: type: QUERY @@ -171,12 +166,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=145) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select PREHOOK: type: QUERY @@ -334,12 +324,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=426) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -434,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=586) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -603,12 +583,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=483) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -762,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=756) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -907,12 +877,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=1113) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1080,12 +1045,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=236) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1195,12 +1155,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out index 3da9284..28f3f5c 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_complex.q.out @@ -144,12 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=789) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -396,12 +391,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=614) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -576,12 +566,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=293) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out index 526cd21..95ea5d9 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_part_all_primitive.q.out @@ -255,12 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=1168) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -456,12 +451,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=1382) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -579,12 +569,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=494) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -686,12 +671,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=407) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -874,12 +854,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=672) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out index 7b420e2..62a7026 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_nonvec_table.q.out @@ -68,12 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=99) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=99) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_permute_select PREHOOK: type: QUERY @@ -181,12 +176,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=114) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=114) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_string_permute_select PREHOOK: type: QUERY @@ -356,12 +346,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=370) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=6 width=370) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -600,12 +585,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=479) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=6 width=479) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -755,12 +735,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=752) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=6 width=752) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out index 86c94f7..0628640 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part.q.out @@ -66,12 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=101) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_permute_select PREHOOK: type: QUERY @@ -171,12 +166,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=145) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select PREHOOK: type: QUERY @@ -334,12 +324,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=426) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -434,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=586) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -603,12 +583,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=483) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -762,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=756) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -907,12 +877,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=1113) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1080,12 +1045,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=236) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1195,12 +1155,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out index 3da9284..28f3f5c 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_complex.q.out @@ -144,12 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=789) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -396,12 +391,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=614) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -576,12 +566,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=293) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out index f155b00..95ea5d9 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_part_all_primitive.q.out @@ -255,12 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=1168) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -456,12 +451,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=1382) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -579,12 +569,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=494) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -686,12 +671,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=407) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -874,12 +854,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=672) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out index ecc4ee6..62a7026 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_orc_vec_table.q.out @@ -68,12 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=99) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=99) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_permute_select PREHOOK: type: QUERY @@ -181,12 +176,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=114) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=114) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_string_permute_select PREHOOK: type: QUERY @@ -356,12 +346,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=370) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=6 width=370) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -600,12 +585,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=479) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=6 width=479) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -755,12 +735,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=752) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=6 width=752) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out index 89a66ba..40219f5 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part.q.out @@ -66,12 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=16) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_permute_select PREHOOK: type: QUERY @@ -171,12 +166,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=19) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select PREHOOK: type: QUERY @@ -334,12 +324,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=94) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -434,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=172) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -603,12 +583,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=182) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -762,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -907,12 +877,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=200) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1080,12 +1045,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=143) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1195,12 +1155,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=71) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out index be42c05..e08305d 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_complex.q.out @@ -144,12 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=155) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -396,12 +391,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=117) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -576,12 +566,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=88) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out index 03c6936..ce0147c 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_part_all_primitive.q.out @@ -255,12 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=512) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -456,12 +451,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=455) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -579,12 +569,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=151) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -686,12 +671,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=76) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -874,12 +854,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=109) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out index da99110..86a0a03 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_nonvec_table.q.out @@ -68,12 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=20) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_permute_select PREHOOK: type: QUERY @@ -181,12 +176,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=21) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=21) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_string_permute_select PREHOOK: type: QUERY @@ -356,12 +346,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=80) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=6 width=80) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -600,12 +585,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=178) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=6 width=178) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -755,12 +735,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=249) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=6 width=249) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out index 1fe9a13..40219f5 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out @@ -66,12 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=16) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_permute_select PREHOOK: type: QUERY @@ -171,12 +166,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=19) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select PREHOOK: type: QUERY @@ -334,12 +324,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=94) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -434,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=172) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -603,12 +583,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=182) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -762,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -907,12 +877,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=200) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1080,12 +1045,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=143) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1195,12 +1155,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=71) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out index be42c05..e08305d 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_complex.q.out @@ -144,12 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=155) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -396,12 +391,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=117) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -576,12 +566,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=88) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out index b8601df..e95f743 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out @@ -255,12 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=507) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -456,12 +451,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=453) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -579,12 +569,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=150) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -686,12 +671,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=76) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -874,12 +854,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=109) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out index 06d2372..86a0a03 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out @@ -68,12 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=20) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_permute_select PREHOOK: type: QUERY @@ -181,12 +176,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=21) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=21) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_string_permute_select PREHOOK: type: QUERY @@ -356,12 +346,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=80) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=6 width=80) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -600,12 +585,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=178) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=6 width=178) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -755,12 +735,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=249) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=6 width=249) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out index 1fe9a13..40219f5 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out @@ -66,12 +66,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=16) - default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_permute_select PREHOOK: type: QUERY @@ -171,12 +166,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=2 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=2 width=19) - default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select PREHOOK: type: QUERY @@ -334,12 +324,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=5 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=94) - default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double PREHOOK: type: QUERY @@ -434,12 +419,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=172) - default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp PREHOOK: type: QUERY @@ -603,12 +583,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=182) - default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -762,12 +737,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=253) - default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY @@ -907,12 +877,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=200) - default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string PREHOOK: type: QUERY @@ -1080,12 +1045,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - TableScan [TS_0] (rows=6 width=143) - default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint,part_change_lower_to_higher_numeric_group_tinyint_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: type: QUERY @@ -1195,12 +1155,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=6 width=71) - default@part_change_lower_to_higher_numeric_group_decimal_to_float,part_change_lower_to_higher_numeric_group_decimal_to_float,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out index be42c05..e08305d 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_complex.q.out @@ -144,12 +144,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=6 width=155) - default@part_change_various_various_struct1,part_change_various_various_struct1,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","s1","b"] + Vertex 1 PREHOOK: query: select insert_num,part,s1,b from part_change_various_various_struct1 PREHOOK: type: QUERY @@ -396,12 +391,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=8 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=8 width=117) - default@part_add_various_various_struct2,part_add_various_various_struct2,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s2"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s2 from part_add_various_various_struct2 PREHOOK: type: QUERY @@ -576,12 +566,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 llap - File Output Operator [FS_2] - Select Operator [SEL_1] (rows=4 width=4) - Output:["_col0","_col1","_col2","_col3"] - TableScan [TS_0] (rows=4 width=88) - default@part_add_to_various_various_struct4,part_add_to_various_various_struct4,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","b","s3"] + Vertex 1 PREHOOK: query: select insert_num,part,b,s3 from part_add_to_various_various_struct4 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out index bb3dd31..ce0147c 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out @@ -255,12 +255,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=10 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"] - TableScan [TS_0] (rows=10 width=512) - default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint PREHOOK: type: QUERY @@ -456,12 +451,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"] - TableScan [TS_0] (rows=6 width=455) - default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double PREHOOK: type: QUERY @@ -579,12 +569,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - TableScan [TS_0] (rows=6 width=151) - default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp PREHOOK: type: QUERY @@ -686,12 +671,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - TableScan [TS_0] (rows=6 width=76) - default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date PREHOOK: type: QUERY @@ -874,12 +854,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=13 width=4) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - TableScan [TS_0] (rows=13 width=109) - default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"] + Vertex 1 PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out index 06d2372..86a0a03 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out @@ -68,12 +68,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=20) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=20) - default@table_add_int_permute_select,table_add_int_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_permute_select PREHOOK: type: QUERY @@ -181,12 +176,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=21) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=6 width=21) - default@table_add_int_string_permute_select,table_add_int_string_permute_select,Tbl:COMPLETE,Col:NONE,Output:["insert_num","a","b"] + Vertex 1 PREHOOK: query: select insert_num,a,b from table_add_int_string_permute_select PREHOOK: type: QUERY @@ -356,12 +346,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=80) - Output:["_col0","_col1","_col2","_col3","_col4"] - TableScan [TS_0] (rows=6 width=80) - default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double PREHOOK: type: QUERY @@ -600,12 +585,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=178) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21"] - TableScan [TS_0] (rows=6 width=178) - default@table_change_numeric_group_string_group_multi_ints_string_group,table_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group PREHOOK: type: QUERY @@ -755,12 +735,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 vectorized, llap - File Output Operator [FS_4] - Select Operator [SEL_3] (rows=6 width=249) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16"] - TableScan [TS_0] (rows=6 width=249) - default@table_change_numeric_group_string_group_floating_string_group,table_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] + Vertex 1 PREHOOK: query: select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out index afb9c4f..4a2c680 100644 --- ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out +++ ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out @@ -11,10 +11,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -35,7 +35,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -704,10 +704,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -727,7 +727,7 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -928,12 +928,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -954,7 +956,22 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -975,23 +992,6 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -2471,10 +2471,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2495,7 +2495,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3164,10 +3164,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3187,7 +3187,7 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3388,12 +3388,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -3414,7 +3416,22 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -3435,23 +3452,6 @@ STAGE PLANS: Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 262 Data size: 46636 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/semijoin.q.out ql/src/test/results/clientpositive/llap/semijoin.q.out index 28fc1d5..f8996ef 100644 --- ql/src/test/results/clientpositive/llap/semijoin.q.out +++ ql/src/test/results/clientpositive/llap/semijoin.q.out @@ -133,11 +133,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -153,31 +153,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -192,7 +168,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -206,6 +182,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -242,11 +242,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -262,31 +262,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -301,7 +277,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -315,6 +291,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -353,11 +353,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -373,31 +373,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -412,7 +388,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -426,6 +402,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -456,11 +456,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -476,31 +476,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 15) (type: boolean) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -519,7 +495,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -533,6 +509,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 15) (type: boolean) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col1 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -574,11 +574,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -594,31 +594,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((value < 'val_10') and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -633,7 +609,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -647,6 +623,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value < 'val_10') and key is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -680,11 +680,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t3 @@ -708,23 +708,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -743,7 +727,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -757,17 +741,33 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t3 + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t3 #### A masked pattern was here #### POSTHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value POSTHOOK: type: QUERY @@ -790,11 +790,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -818,23 +818,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -853,7 +837,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -867,6 +851,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -897,11 +897,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -925,23 +925,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -956,7 +940,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -970,6 +954,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1005,11 +1005,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1024,31 +1024,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1063,7 +1039,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1077,6 +1053,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1126,11 +1126,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1146,31 +1146,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (2 * key) is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: (2 * _col0) (type: int) - sort order: + - Map-reduce partition columns: (2 * _col0) (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1185,7 +1161,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 12 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1199,6 +1175,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (2 * key) is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (2 * _col0) (type: int) + sort order: + + Map-reduce partition columns: (2 * _col0) (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1233,11 +1233,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1253,7 +1253,43 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: int), _col3 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1269,7 +1305,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1293,42 +1329,6 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: int), _col3 (type: string) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1375,11 +1375,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1394,31 +1394,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1433,7 +1409,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1447,6 +1423,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1491,11 +1491,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1510,7 +1510,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Semi Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1534,7 +1565,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1558,37 +1589,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Left Semi Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1635,11 +1635,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1651,7 +1651,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1663,7 +1694,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1684,37 +1715,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1773,11 +1773,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1789,7 +1789,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1801,7 +1832,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1822,37 +1853,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1914,11 +1914,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1930,7 +1930,38 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1942,7 +1973,7 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -1963,37 +1994,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Semi Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2055,56 +2055,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: c - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2121,7 +2088,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2135,6 +2102,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2196,11 +2196,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2212,7 +2212,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2233,7 +2264,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2245,37 +2276,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Right Outer Join0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2339,11 +2339,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2355,7 +2355,38 @@ STAGE PLANS: Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + keys: + 0 key (type: int) + 1 _col0 (type: int) + 2 key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -2376,7 +2407,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -2388,37 +2419,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - Outer Join 0 to 2 - keys: - 0 key (type: int) - 1 _col0 (type: int) - 2 key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 48 Data size: 358 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2493,12 +2493,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2514,43 +2514,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2567,7 +2531,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 24 Data size: 179 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2582,7 +2546,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 196 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -2596,6 +2560,42 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2663,10 +2663,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2686,7 +2686,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2710,24 +2728,6 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 92 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/skewjoin.q.out ql/src/test/results/clientpositive/llap/skewjoin.q.out index 1251b67..5a41ea1 100644 --- ql/src/test/results/clientpositive/llap/skewjoin.q.out +++ ql/src/test/results/clientpositive/llap/skewjoin.q.out @@ -89,10 +89,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src1 @@ -111,27 +111,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: src2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -154,6 +134,26 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 + Vertex 3 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -213,10 +213,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -232,7 +232,33 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + 3 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -248,7 +274,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -264,7 +290,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: d @@ -280,32 +306,6 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - Inner Join 2 to 3 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - 3 key (type: string) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -355,10 +355,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -374,7 +374,33 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + 3 key (type: string) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -390,7 +416,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -406,7 +432,7 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: d @@ -422,32 +448,6 @@ STAGE PLANS: value expressions: val (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - Inner Join 2 to 3 - keys: - 0 key (type: string) - 1 key (type: string) - 2 key (type: string) - 3 key (type: string) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -489,11 +489,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -509,23 +509,7 @@ STAGE PLANS: value expressions: key (type: string), val (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key + 1) is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (key + 1) (type: double) - sort order: + - Map-reduce partition columns: (key + 1) (type: double) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: key (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -545,7 +529,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -560,6 +544,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: key (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -601,11 +601,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -624,27 +624,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -668,7 +648,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -683,6 +663,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -732,11 +732,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -755,27 +755,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -799,7 +779,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -814,6 +794,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -871,11 +871,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -894,46 +894,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) < 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -959,7 +920,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -974,6 +935,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1021,11 +1021,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: k @@ -1038,20 +1038,7 @@ STAGE PLANS: value expressions: key (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: v - Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE - value expressions: val (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1071,7 +1058,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1086,6 +1073,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: v + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: val (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out index 5d7f371..6fd3de7 100644 --- ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out +++ ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -104,7 +104,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -124,24 +142,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -182,10 +182,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -202,24 +202,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -237,6 +220,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -279,11 +279,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -302,26 +302,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -340,7 +321,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -355,6 +336,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -388,11 +388,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -408,23 +408,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -443,7 +427,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -458,6 +442,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_cache.q.out ql/src/test/results/clientpositive/llap/smb_cache.q.out index 932b211..85b17f0 100644 --- ql/src/test/results/clientpositive/llap/smb_cache.q.out +++ ql/src/test/results/clientpositive/llap/smb_cache.q.out @@ -203,34 +203,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=579 width=8) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=130 width=8) - Output:["_col0"] - Filter Operator [FIL_13] (rows=130 width=8) - predicate:userid is not null - TableScan [TS_0] (rows=130 width=8) - default@bug_201_input_b,t1,Tbl:COMPLETE,Col:NONE,Output:["userid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=527 width=8) - Output:["_col0"] - Filter Operator [FIL_14] (rows=527 width=8) - predicate:userid is not null - TableScan [TS_3] (rows=527 width=8) - default@bug_201_input_a,fa,Tbl:COMPLETE,Col:NONE,Output:["userid"] + Vertex 2 PREHOOK: query: select t1.userid, @@ -283,34 +262,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=579 width=8) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=130 width=8) - Output:["_col0"] - Filter Operator [FIL_13] (rows=130 width=8) - predicate:userid is not null - TableScan [TS_0] (rows=130 width=8) - default@bug_201_input_b,t1,Tbl:COMPLETE,Col:NONE,Output:["userid"] - <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=527 width=8) - Output:["_col0"] - Filter Operator [FIL_14] (rows=527 width=8) - predicate:userid is not null - TableScan [TS_3] (rows=527 width=8) - default@bug_201_input_a,fa,Tbl:COMPLETE,Col:NONE,Output:["userid"] + Vertex 2 PREHOOK: query: select t1.userid, diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out index 8dee9ff..72d9f8b 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_14.q.out @@ -57,11 +57,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -76,22 +76,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -110,7 +95,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -125,6 +110,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -172,12 +172,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -192,22 +192,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -230,7 +215,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -244,7 +229,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -258,6 +243,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -320,12 +320,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -340,22 +340,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -378,7 +363,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -398,7 +383,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -413,6 +398,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -468,30 +468,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -510,7 +491,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -529,7 +510,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -544,6 +525,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -603,11 +603,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -626,22 +626,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -660,7 +645,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -675,6 +660,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -758,30 +758,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((key < 8) and (key < 6)) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -800,7 +781,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -819,7 +800,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -834,6 +815,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 8) and (key < 6)) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -905,11 +905,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -928,26 +928,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key < 8) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -966,7 +947,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -981,6 +962,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1030,11 +1030,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1053,26 +1053,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (key + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1091,7 +1072,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1106,6 +1087,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (key + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1151,11 +1151,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1174,22 +1174,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1208,7 +1193,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1223,6 +1208,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1264,11 +1264,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1287,22 +1287,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1321,7 +1306,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1336,6 +1321,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1387,11 +1387,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1410,7 +1410,43 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: a @@ -1429,7 +1465,7 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: a @@ -1448,42 +1484,6 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1551,11 +1551,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1574,22 +1574,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1608,7 +1593,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1623,6 +1608,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out index 20766c2..dcc7b06 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out @@ -47,11 +47,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -127,7 +127,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types int:string:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -203,66 +263,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types int:string:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -357,11 +357,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -437,7 +437,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -513,66 +573,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int), key2 (type: int) - 1 key (type: int), key2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -615,11 +615,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -695,7 +695,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key2 (type: int), key (type: int) + 1 key2 (type: int), key (type: int) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -771,66 +831,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key2 (type: int), key (type: int) - 1 key2 (type: int), key (type: int) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -873,11 +873,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -953,7 +953,67 @@ STAGE PLANS: name: default.test_table1 Truncated Path -> Alias: /test_table1 [a] - Map 4 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int), value (type: string) + 1 key (type: int), value (type: string) + outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + Position of Big Table: 0 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + auto parallelism: false + Vertex 3 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int:int:string:int:int:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Vertex 4 Map Operator Tree: TableScan alias: b @@ -1029,66 +1089,6 @@ STAGE PLANS: name: default.test_table2 Truncated Path -> Alias: /test_table2 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int), value (type: string) - 1 key (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 - Position of Big Table: 0 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - tag: -1 - TopN: 10 - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) - auto parallelism: false - Reducer 3 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 550 Data size: 7939 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 10 Data size: 140 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int:int:string:int:int:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out index a97a692..4b95abe 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out @@ -185,11 +185,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -204,7 +204,51 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + Inner Join 0 to 3 + Inner Join 0 to 4 + Inner Join 0 to 5 + Inner Join 0 to 6 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + 3 key (type: int) + 4 key (type: int) + 5 key (type: int) + 6 key (type: int) + Statistics: Num rows: 66 Data size: 462 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -219,7 +263,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -234,7 +278,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: d @@ -249,7 +293,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: e @@ -264,7 +308,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: f @@ -279,7 +323,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 9 + Vertex 9 Map Operator Tree: TableScan alias: g @@ -294,50 +338,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - Inner Join 0 to 3 - Inner Join 0 to 4 - Inner Join 0 to 5 - Inner Join 0 to 6 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - 3 key (type: int) - 4 key (type: int) - 5 key (type: int) - 6 key (type: int) - Statistics: Num rows: 66 Data size: 462 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -405,10 +405,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -492,7 +492,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -576,10 +576,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -673,7 +673,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -787,10 +787,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -962,7 +962,7 @@ STAGE PLANS: Statistics: Num rows: 165 Data size: 1155 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: llap - Map 18 + Vertex 18 Map Operator Tree: TableScan alias: q @@ -978,7 +978,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 19 + Vertex 19 Map Operator Tree: TableScan alias: r @@ -994,7 +994,31 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 20 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join0 to 2 + Left Outer Join0 to 3 + Left Outer Join0 to 4 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + 3 _col0 (type: int) + 4 _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 20 Map Operator Tree: TableScan alias: s @@ -1010,7 +1034,7 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 21 + Vertex 21 Map Operator Tree: TableScan alias: t @@ -1026,30 +1050,6 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join0 to 2 - Left Outer Join0 to 3 - Left Outer Join0 to 4 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - 3 _col0 (type: int) - 4 _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 726 Data size: 5082 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out index e2d5e4e..12be0b2 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -226,10 +226,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -249,7 +249,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -369,7 +369,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out index c3b29ea..519b106 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out @@ -49,7 +49,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out index 7d7d96d..f999962 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -80,7 +80,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -96,7 +120,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -112,30 +136,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -170,10 +170,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -186,7 +186,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -199,7 +223,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -212,30 +236,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -270,10 +270,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -286,7 +286,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -299,7 +323,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -312,30 +336,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -391,7 +391,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -404,7 +428,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -417,30 +441,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -481,10 +481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -497,7 +497,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -510,7 +534,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -523,30 +547,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -592,10 +592,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -608,7 +608,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -621,7 +645,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -634,30 +658,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -694,10 +694,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -710,7 +710,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -723,7 +747,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -736,30 +760,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -810,37 +810,11 @@ STAGE PLANS: key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -864,6 +838,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -904,10 +904,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -920,7 +920,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -933,7 +957,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -946,30 +970,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1012,10 +1012,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1028,7 +1028,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1041,7 +1065,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1054,30 +1078,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1114,10 +1114,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1130,7 +1130,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1143,7 +1167,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1156,30 +1180,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1223,10 +1223,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1239,7 +1239,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1252,7 +1276,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1265,30 +1289,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1329,10 +1329,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1345,7 +1345,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1358,7 +1382,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1371,30 +1395,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out index 1abe220..cfc9ef3 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -80,7 +80,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -96,7 +120,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -112,30 +136,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -170,10 +170,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -186,7 +186,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -199,7 +223,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -212,30 +236,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -270,10 +270,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -286,7 +286,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -299,7 +323,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -312,30 +336,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -375,10 +375,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -391,7 +391,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -404,7 +428,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -417,30 +441,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -481,10 +481,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -497,7 +497,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -510,7 +534,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -523,30 +547,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -592,10 +592,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -608,7 +608,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -621,7 +645,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -634,30 +658,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -694,10 +694,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -710,7 +710,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -723,7 +747,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -736,30 +760,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -810,37 +810,11 @@ STAGE PLANS: key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -864,6 +838,32 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 2 Data size: 222 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -904,10 +904,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -920,7 +920,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -933,7 +957,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -946,30 +970,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Right Outer Join0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1012,10 +1012,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1028,7 +1028,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1041,7 +1065,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1054,30 +1078,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1114,10 +1114,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1130,7 +1130,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1143,7 +1167,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1156,30 +1180,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Left Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1223,10 +1223,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1239,7 +1239,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Right Outer Join1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1252,7 +1276,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1265,30 +1289,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Right Outer Join1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1329,10 +1329,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1345,7 +1345,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Outer Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -1358,7 +1382,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1371,30 +1395,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Outer Join 0 to 1 - Outer Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 457 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out index bbf9b46..6440616 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out @@ -73,10 +73,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -92,23 +92,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -131,6 +115,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -1251,10 +1251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1270,23 +1270,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1309,6 +1293,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2445,10 +2445,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2464,23 +2464,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2503,6 +2487,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2555,10 +2555,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2574,23 +2574,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key > 1000) (type: boolean) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: int) - sort order: + - Map-reduce partition columns: key (type: int) - Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2613,6 +2597,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.smb_join_results + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: int) + sort order: + + Map-reduce partition columns: key (type: int) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection @@ -2661,10 +2661,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -2680,7 +2680,31 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -2696,7 +2720,7 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -2712,30 +2736,6 @@ STAGE PLANS: value expressions: value (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 1 to 2 - keys: - 0 key (type: int) - 1 key (type: int) - 2 key (type: int) - outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 365 Data size: 3513 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/sqlmerge.q.out ql/src/test/results/clientpositive/llap/sqlmerge.q.out index 486e812..2134a98 100644 --- ql/src/test/results/clientpositive/llap/sqlmerge.q.out +++ ql/src/test/results/clientpositive/llap/sqlmerge.q.out @@ -41,14 +41,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Reducer 2 (SIMPLE_EDGE) - Reducer 6 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -61,20 +61,7 @@ STAGE PLANS: value expressions: ROW__ID (type: struct) Execution mode: llap LLAP IO: may be used (ACID table) - Map 7 - Map Operator Tree: - TableScan - alias: s - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: a (type: int) - sort order: + - Map-reduce partition columns: a (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: b (type: int) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -141,7 +128,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: int), _col1 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -156,7 +143,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.acidtbl - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -171,7 +158,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.acidtbl - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -195,7 +182,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_tmp_table - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -210,6 +197,19 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.acidtbl + Vertex 7 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: a (type: int) + sort order: + + Map-reduce partition columns: a (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: b (type: int) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-5 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/stats11.q.out ql/src/test/results/clientpositive/llap/stats11.q.out index 3eccf52..d799acd 100644 --- ql/src/test/results/clientpositive/llap/stats11.q.out +++ ql/src/test/results/clientpositive/llap/stats11.q.out @@ -313,10 +313,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -388,7 +388,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -462,55 +511,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 0 - numRows 0 - rawDataSize 0 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 0 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection @@ -671,10 +671,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -746,7 +746,56 @@ STAGE PLANS: name: default.srcbucket_mapjoin Truncated Path -> Alias: /srcbucket_mapjoin [a] - Map 3 + Vertex 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col6 + Position of Big Table: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value1,value2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.bucketmapjoin_tmp_result + numFiles 1 + numRows 464 + rawDataSize 8519 + serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 8983 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketmapjoin_tmp_result + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Vertex 3 Map Operator Tree: TableScan alias: b @@ -820,55 +869,6 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Truncated Path -> Alias: /srcbucket_mapjoin_part/ds=2008-04-08 [b] - Reducer 2 - Execution mode: llap - Needs Tagging: false - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0, _col1, _col6 - Position of Big Table: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 60 Data size: 6877 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} - bucket_count -1 - column.name.delimiter , - columns key,value1,value2 - columns.comments - columns.types string:string:string -#### A masked pattern was here #### - name default.bucketmapjoin_tmp_result - numFiles 1 - numRows 464 - rawDataSize 8519 - serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 8983 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketmapjoin_tmp_result - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out index 8a85bf4..1bff6aa 100644 --- ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out +++ ql/src/test/results/clientpositive/llap/stats_based_fetch_decision.q.out @@ -101,7 +101,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -142,7 +142,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcpart diff --git ql/src/test/results/clientpositive/llap/stats_only_null.q.out ql/src/test/results/clientpositive/llap/stats_only_null.q.out index c905ceb..945755f 100644 --- ql/src/test/results/clientpositive/llap/stats_only_null.q.out +++ ql/src/test/results/clientpositive/llap/stats_only_null.q.out @@ -81,10 +81,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_null @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -141,10 +141,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: stats_null_part @@ -164,7 +164,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/subquery_exists.q.out ql/src/test/results/clientpositive/llap/subquery_exists.q.out index 14b592b..07b6694 100644 --- ql/src/test/results/clientpositive/llap/subquery_exists.q.out +++ ql/src/test/results/clientpositive/llap/subquery_exists.q.out @@ -25,13 +25,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -47,7 +47,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -66,42 +84,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -122,7 +105,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -135,7 +118,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -291,13 +291,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -314,40 +314,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -365,7 +332,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -386,7 +369,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -399,7 +382,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -419,7 +419,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from src b @@ -447,11 +447,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -466,7 +466,25 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -488,25 +506,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -526,7 +526,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src b where exists @@ -1069,7 +1069,7 @@ POSTHOOK: query: insert into t values(0) POSTHOOK: type: QUERY POSTHOOK: Output: default@t POSTHOOK: Lineage: t.i EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from t where exists (select count(*) from src where 1=2) PREHOOK: type: QUERY POSTHOOK: query: explain select * from t where exists (select count(*) from src where 1=2) @@ -1083,11 +1083,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -1102,7 +1102,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1125,25 +1143,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 12 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1170,7 +1170,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from t where exists (select count(*) from src where 1=2) PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git ql/src/test/results/clientpositive/llap/subquery_in.q.out ql/src/test/results/clientpositive/llap/subquery_in.q.out index 63432a0..eda1f23 100644 --- ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -17,11 +17,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: s1 @@ -61,25 +79,7 @@ STAGE PLANS: Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -149,13 +149,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -171,7 +171,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -191,42 +209,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -247,7 +230,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -260,7 +243,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -338,13 +338,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -364,20 +364,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -395,7 +382,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -439,7 +439,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -460,7 +460,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -527,15 +527,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -552,37 +552,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -604,7 +574,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -645,7 +628,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -672,7 +655,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -691,7 +674,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -708,7 +691,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -779,13 +779,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -801,7 +801,25 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string), _col1 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: a @@ -821,42 +839,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -877,7 +860,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -895,7 +878,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -987,13 +987,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -1013,51 +1013,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 9200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: l_orderkey - Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: l_orderkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1070,7 +1026,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1087,7 +1043,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1109,7 +1065,51 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 9200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean) + Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int) + outputColumnNames: l_orderkey + Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: l_orderkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1169,7 +1169,7 @@ POSTHOOK: Input: default@lineitem #### A masked pattern was here #### 108570 8571 4297 1798 -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 11' is a cross product PREHOOK: query: explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY POSTHOOK: query: explain select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) @@ -1183,17 +1183,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (CUSTOM_SIMPLE_EDGE), Map 13 (CUSTOM_SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (PARTITION_EDGE), Vertex 13 (PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -1213,7 +1213,7 @@ STAGE PLANS: value expressions: _col1 (type: double), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: lineitem @@ -1225,58 +1225,7 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_quantity (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1297,7 +1246,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1310,7 +1259,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 23 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1327,7 +1291,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: double), _col3 (type: int) Statistics: Num rows: 26 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1347,7 +1311,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: double) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1362,7 +1326,43 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_quantity (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1389,7 +1389,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 924 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1408,7 +1408,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: int) Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1428,7 +1428,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product +Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_3, $hdt$_4]] in Stage 'Vertex 11' is a cross product PREHOOK: query: select sum(l_extendedprice) from lineitem, part where p_partkey = l_partkey and l_quantity IN (select avg(l_quantity) from lineitem where l_partkey = p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@lineitem @@ -1453,13 +1453,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1479,41 +1479,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1531,7 +1497,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1556,7 +1539,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1582,7 +1565,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1647,12 +1647,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1672,7 +1672,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (_col5 - 1) (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -1695,25 +1713,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col5 - 1) (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1739,7 +1739,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1790,12 +1790,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -1815,7 +1815,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (_col0 * _col5) (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -1838,25 +1856,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col0 * _col5) (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1882,7 +1882,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1924,14 +1924,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -1947,41 +1947,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2000,7 +1966,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2015,7 +1981,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2040,7 +2023,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2057,7 +2040,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2099,12 +2099,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2124,7 +2124,25 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 floor(_col7) (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -2147,25 +2165,7 @@ STAGE PLANS: value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 floor(_col7) (type: bigint) - 1 _col0 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 3714 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2195,7 +2195,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2261,13 +2261,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2284,41 +2284,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2336,7 +2302,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2357,7 +2340,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2374,7 +2357,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2417,13 +2417,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2440,41 +2440,7 @@ STAGE PLANS: value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2492,7 +2458,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2517,7 +2500,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2534,7 +2517,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2601,13 +2601,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2624,41 +2624,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2676,7 +2642,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2697,7 +2680,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2710,7 +2693,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2777,13 +2777,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2800,45 +2800,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2856,7 +2818,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2877,7 +2856,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2890,7 +2869,28 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2957,15 +2957,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -2985,51 +2985,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3052,7 +3008,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3070,7 +3026,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3088,7 +3044,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3101,7 +3077,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3127,7 +3127,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3199,17 +3199,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3225,7 +3225,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -3249,41 +3249,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: sc - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3309,7 +3275,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3322,7 +3288,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3345,7 +3311,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3363,7 +3329,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3381,7 +3347,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: sc + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3406,7 +3389,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3423,7 +3406,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3495,13 +3495,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3521,30 +3521,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3560,7 +3537,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3574,7 +3551,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3600,7 +3600,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3651,13 +3651,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3677,30 +3677,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3717,7 +3694,7 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -3734,7 +3711,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3760,7 +3760,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3805,13 +3805,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Union 4 (CONTAINS) - Map 6 <- Union 4 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Union 4 (CONTAINS) + Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 4 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -3831,7 +3833,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: part @@ -3855,7 +3875,20 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 6 Map Operator Tree: TableScan alias: part @@ -3879,39 +3912,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -3944,14 +3944,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -3968,44 +3968,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_size < 10) (type: boolean) - Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_mfgr (type: string), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4025,7 +3988,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -4039,7 +4002,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (p_size < 10) (type: boolean) + Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_mfgr (type: string), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4064,7 +4047,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4081,15 +4064,32 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) @@ -4132,14 +4132,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4156,7 +4156,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col2 (type: int) + 1 _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: pp @@ -4176,62 +4194,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string), _col2 (type: int) - 1 _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4248,7 +4211,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4273,7 +4236,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4286,7 +4249,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4299,6 +4279,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4355,16 +4355,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4381,44 +4381,24 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: int) + Vertex 11 Map Operator Tree: TableScan alias: part @@ -4435,44 +4415,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4485,7 +4428,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4503,7 +4446,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 26 Data size: 5954 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4520,7 +4483,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 18 Data size: 4050 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4541,7 +4504,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4558,7 +4521,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 8 Data size: 1032 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4571,15 +4551,35 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select p_partkey from part where p_name in - (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size and p.p_size=part.p_size) + Vertex 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select p_partkey from part where p_name in + (select p.p_name from part p left outer join part pp on p.p_type = pp.p_type where pp.p_size = part.p_size and p.p_size=part.p_size) PREHOOK: type: QUERY PREHOOK: Input: default@part #### A masked pattern was here #### @@ -4629,15 +4629,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4654,7 +4654,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -4675,64 +4675,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4750,7 +4693,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4768,7 +4711,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4791,7 +4754,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 3 Data size: 687 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4804,7 +4767,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 3 Data size: 687 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4821,7 +4801,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4860,14 +4860,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4884,7 +4884,25 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col2 (type: int) + 1 _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: pp @@ -4904,62 +4922,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string), _col2 (type: int) - 1 _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4976,7 +4939,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 18 Data size: 1944 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5001,7 +4964,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5014,7 +4977,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5027,6 +5007,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5083,18 +5083,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5111,73 +5111,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5200,7 +5134,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5219,7 +5153,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5236,7 +5170,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: bigint) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5249,7 +5200,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5273,7 +5224,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), UDFToLong(_col5) (type: bigint) Statistics: Num rows: 8 Data size: 5016 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5291,7 +5242,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5312,7 +5279,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5332,7 +5299,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5345,6 +5329,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5376,14 +5376,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5400,41 +5400,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5452,7 +5418,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5479,7 +5462,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5498,7 +5481,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: double) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5515,7 +5498,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: int), _col0 (type: double) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5569,7 +5569,7 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: explain select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) @@ -5583,14 +5583,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5607,39 +5607,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5657,7 +5625,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5687,7 +5670,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5706,7 +5689,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5723,7 +5706,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: int), _col0 (type: int) Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5742,7 +5742,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: select * from part where p_size in (select min(pp.p_size) from part pp where pp.p_partkey > part.p_partkey) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5767,24 +5767,24 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 19 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5801,7 +5801,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: pp @@ -5817,107 +5817,7 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5940,7 +5840,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5961,7 +5861,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5974,7 +5891,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5997,7 +5930,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6016,7 +5949,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: string) Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6034,7 +5967,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6051,7 +5984,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: string) Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6075,7 +6008,24 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 8 Data size: 5016 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6088,7 +6038,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6102,7 +6069,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToLong(_col0) (type: bigint) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6119,7 +6086,7 @@ STAGE PLANS: Map-reduce partition columns: _col5 (type: int), _col4 (type: string) Statistics: Num rows: 8 Data size: 5080 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6144,7 +6111,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6165,7 +6148,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6185,7 +6168,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6229,179 +6229,78 @@ POSTHOOK: Input: default@part 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful -48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i -49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick -65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr -78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull -86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully -90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -PREHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 19 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +PREHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from part where p_size not in (select avg(pp.p_size) from part pp where pp.p_partkey = part.p_partkey) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 10 Map Operator Tree: TableScan alias: pp - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6428,7 +6327,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6449,7 +6348,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6462,7 +6378,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6489,7 +6422,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6508,7 +6441,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: int) Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6526,7 +6459,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6543,7 +6476,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col3 (type: int) Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6567,7 +6500,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8 Data size: 5016 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6580,7 +6530,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6594,7 +6561,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6611,7 +6578,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col5 (type: int) Statistics: Num rows: 8 Data size: 5080 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6636,7 +6603,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6657,7 +6640,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6677,7 +6660,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6748,12 +6748,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -6773,25 +6773,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6809,7 +6791,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6830,7 +6830,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6975,14 +6975,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -6999,41 +6999,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: tt - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7051,7 +7017,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: tt + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7078,7 +7061,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7097,7 +7080,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7110,7 +7093,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: bigint), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/subquery_multi.q.out ql/src/test/results/clientpositive/llap/subquery_multi.q.out index bbdce1d..44c71e3 100644 --- ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -88,13 +88,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -114,47 +114,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_size is not null (type: boolean) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -171,7 +131,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -189,7 +149,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -202,7 +182,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_brand is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -257,7 +257,7 @@ POSTHOOK: Input: default@part_null 78487 NULL Manufacturer#6 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) @@ -271,15 +271,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -299,64 +299,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_name) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -374,7 +317,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -389,7 +332,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -406,7 +349,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -431,7 +374,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -444,7 +407,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_name) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -456,6 +439,23 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -463,7 +463,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -472,7 +472,7 @@ POSTHOOK: query: select * from part_null where p_name IN (select p_name from par POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) @@ -486,15 +486,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -514,64 +514,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_type) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -589,7 +532,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -604,7 +547,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -621,7 +564,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -646,9 +589,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Reduce Operator Tree: + LLAP IO: no inputs + Vertex 6 + Execution mode: llap + Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial @@ -659,7 +622,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_type) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -671,6 +654,23 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -678,7 +678,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -713,7 +713,7 @@ POSTHOOK: Input: default@part_null 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) @@ -727,15 +727,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -755,67 +755,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(p_name) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -833,7 +773,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -848,7 +788,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -865,7 +805,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 17 Data size: 2079 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -890,7 +830,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_brand is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -903,7 +863,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(p_name) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -915,6 +895,26 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -922,7 +922,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_brand IN (select p_brand from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -931,7 +931,7 @@ POSTHOOK: query: select * from part_null where p_brand IN (select p_brand from p POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) @@ -945,15 +945,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -971,64 +971,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c (type: char(2)) - outputColumnNames: c - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: count(), count(c) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: c (type: char(2)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(2)) - sort order: + - Map-reduce partition columns: _col0 (type: char(2)) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1041,7 +984,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1058,7 +1001,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 5 Data size: 3341 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1082,7 +1025,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 2 Data size: 1470 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1100,7 +1043,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: c (type: char(2)) + outputColumnNames: c + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: count(), count(c) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1112,7 +1075,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: c (type: char(2)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(2)) + sort order: + + Map-reduce partition columns: _col0 (type: char(2)) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1130,22 +1110,42 @@ STAGE PLANS: Map-reduce partition columns: UDFToString(_col0) (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: boolean) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) -PREHOOK: type: QUERY -PREHOOK: Input: default@part_null -PREHOOK: Input: default@tempty -#### A masked pattern was here #### -POSTHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@part_null + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_brand is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +PREHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) +PREHOOK: type: QUERY +PREHOOK: Input: default@part_null +PREHOOK: Input: default@tempty +#### A masked pattern was here #### +POSTHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_null POSTHOOK: Input: default@tempty #### A masked pattern was here #### 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr @@ -1175,7 +1175,7 @@ POSTHOOK: Input: default@tempty 78487 NULL Manufacturer#6 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) @@ -1189,13 +1189,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1215,46 +1215,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tnull - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1269,7 +1230,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1287,7 +1248,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1300,7 +1281,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tnull + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1320,7 +1320,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND EXISTS (select c from tnull) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1357,7 +1357,7 @@ POSTHOOK: Input: default@tnull 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) @@ -1371,13 +1371,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1397,46 +1397,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_size is not null (type: boolean) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1451,7 +1412,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1469,7 +1430,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1482,7 +1463,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1502,7 +1502,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_size IN (select p_size from part_null) AND EXISTS (select c from tempty) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1513,7 +1513,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null POSTHOOK: Input: default@tempty #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) @@ -1527,13 +1527,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1553,46 +1553,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: tempty - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1607,7 +1568,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 17 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1632,7 +1593,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1645,7 +1626,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: tempty + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: true (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1664,7 +1664,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND NOT EXISTS (select c from tempty) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1701,8 +1701,8 @@ POSTHOOK: Input: default@tempty 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) @@ -1716,35 +1716,35 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (CUSTOM_SIMPLE_EDGE), Reducer 21 (CUSTOM_SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (CUSTOM_SIMPLE_EDGE), Reducer 26 (CUSTOM_SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE) - Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 40 (SIMPLE_EDGE) - Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) - Reducer 34 <- Reducer 33 (SIMPLE_EDGE) - Reducer 36 <- Map 35 (SIMPLE_EDGE) - Reducer 38 <- Map 37 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) - Reducer 40 <- Map 39 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (PARTITION_EDGE), Vertex 21 (PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (PARTITION_EDGE), Vertex 26 (PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) + Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) + Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) + Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) + Vertex 40 <- Vertex 39 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -1761,7 +1761,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -1778,215 +1778,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 27 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 32 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 35 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 37 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 39 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2003,7 +1795,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2030,7 +1822,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2045,7 +1837,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2070,7 +1879,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2087,7 +1896,22 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2108,7 +1932,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2121,7 +1945,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2138,7 +1962,28 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 6 Data size: 1485 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2151,7 +1996,22 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2172,7 +2032,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2185,7 +2045,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2198,7 +2079,24 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2215,7 +2113,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2236,7 +2134,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2253,7 +2151,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 6 Data size: 1633 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2271,7 +2169,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 31 + Vertex 31 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2288,7 +2186,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 33 + Vertex 32 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 33 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2313,7 +2228,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2330,7 +2245,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 36 + Vertex 35 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 36 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2343,7 +2275,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 38 + Vertex 37 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 38 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2356,7 +2305,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 39 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2381,7 +2347,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 40 + Vertex 40 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2394,7 +2360,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2419,7 +2402,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2436,7 +2419,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2456,8 +2456,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2507,17 +2507,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -2537,84 +2537,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 12 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2635,7 +2558,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 17 Data size: 3581 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2648,7 +2571,27 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1685 Basic stats: COMPLETE Column stats: NONE - Reducer 13 + Vertex 12 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2661,7 +2604,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2678,7 +2621,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2696,7 +2639,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2709,7 +2672,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2730,7 +2710,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 28 Data size: 5605 Basic stats: COMPLETE Column stats: NONE - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2743,6 +2723,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 14 Data size: 2802 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2786,8 +2786,8 @@ POSTHOOK: Input: default@part_null 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu 155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) @@ -2795,164 +2795,58 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (CUSTOM_SIMPLE_EDGE), Reducer 21 (CUSTOM_SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 21 <- Map 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (CUSTOM_SIMPLE_EDGE), Reducer 26 (CUSTOM_SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE) - Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 40 (SIMPLE_EDGE) - Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) - Reducer 34 <- Reducer 33 (SIMPLE_EDGE) - Reducer 36 <- Map 35 (SIMPLE_EDGE) - Reducer 38 <- Map 37 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) - Reducer 40 <- Map 39 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) - sort order: +++ - Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) - Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 20 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_container (type: string) - outputColumnNames: p_type, p_container - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (PARTITION_EDGE), Vertex 21 (PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (PARTITION_EDGE), Vertex 26 (PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) + Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) + Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) + Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) + Vertex 40 <- Vertex 39 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + alias: part_null + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_container (type: string) - outputColumnNames: p_type, p_container - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) + sort order: +++ + Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) + Statistics: Num rows: 5 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 27 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -2969,109 +2863,7 @@ STAGE PLANS: value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 32 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 35 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 37 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 39 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_container (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3088,7 +2880,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3115,7 +2907,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3130,7 +2922,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3155,7 +2964,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3172,7 +2981,22 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3193,7 +3017,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3206,7 +3030,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3223,7 +3047,28 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string) Statistics: Num rows: 5 Data size: 3581 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 21 + Vertex 20 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_container (type: string) + outputColumnNames: p_type, p_container + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3236,7 +3081,22 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3257,7 +3117,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3270,7 +3130,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 28 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_container (type: string) + outputColumnNames: p_type, p_container + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3283,7 +3164,24 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3300,7 +3198,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3321,7 +3219,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3338,7 +3236,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col3 (type: string) Statistics: Num rows: 5 Data size: 3939 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col13 (type: bigint), _col14 (type: bigint) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3356,7 +3254,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 31 + Vertex 31 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3373,7 +3271,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: string) Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 33 + Vertex 32 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 33 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3398,7 +3313,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3415,7 +3330,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 36 + Vertex 35 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 36 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3428,7 +3360,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 38 + Vertex 37 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 38 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3441,7 +3390,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 39 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3466,7 +3432,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 40 + Vertex 40 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3479,7 +3445,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3504,7 +3487,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 317 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3521,7 +3504,24 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 317 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string), p_container (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3541,8 +3541,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product -Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[187][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 18' is a cross product +Warning: Shuffle Join MERGEJOIN[190][tables = [$hdt$_4, $hdt$_5]] in Stage 'Vertex 23' is a cross product PREHOOK: query: select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3592,23 +3592,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE) - Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 20 <- Map 19 (SIMPLE_EDGE) - Reducer 22 <- Map 21 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 18 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -3625,41 +3625,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -3676,75 +3642,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 21 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part_null - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3771,7 +3669,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3786,7 +3684,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 223 Data size: 892 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3799,7 +3714,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3820,7 +3752,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 447 Data size: 1790 Basic stats: COMPLETE Column stats: NONE - Reducer 17 + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3838,7 +3770,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 223 Data size: 892 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 18 + Vertex 18 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3855,7 +3787,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: int) Statistics: Num rows: 245 Data size: 981 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 2 + Vertex 19 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 814 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3872,7 +3821,7 @@ STAGE PLANS: Map-reduce partition columns: _col5 (type: int) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3885,7 +3834,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 407 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 22 + Vertex 21 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 22 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3898,7 +3864,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3915,7 +3881,7 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col5 (type: int) Statistics: Num rows: 245 Data size: 981 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3940,7 +3906,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3961,7 +3944,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 28 Data size: 6435 Basic stats: COMPLETE Column stats: NONE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3974,7 +3957,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 14 Data size: 3217 Basic stats: COMPLETE Column stats: NONE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part_null + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 32 Data size: 3256 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4043,17 +4043,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 13 <- Map 12 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part_null @@ -4070,7 +4070,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -4087,7 +4087,20 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 12 + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 12 Map Operator Tree: TableScan alias: part @@ -4104,54 +4117,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col2 (type: string), _col1 (type: string) - Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: pp - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4164,7 +4130,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4182,7 +4148,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col1 (type: string) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4199,7 +4182,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4220,7 +4203,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4233,7 +4216,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 225 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4258,7 +4258,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4318,7 +4318,7 @@ POSTHOOK: Input: default@part_null 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 8' is a cross product PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4338,22 +4338,22 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 13 <- Map 15 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (CUSTOM_SIMPLE_EDGE) - Reducer 19 <- Map 18 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 8 <- Reducer 17 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Reducer 20 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (PARTITION_EDGE) + Vertex 19 <- Vertex 18 (PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 17 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 9 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: lineitem @@ -4373,7 +4373,20 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: lineitem @@ -4393,120 +4406,7 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: l_partkey is not null (type: boolean) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_partkey (type: int), l_linenumber (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_quantity (type: double) - outputColumnNames: l_quantity - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(l_quantity) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_quantity (type: double) - outputColumnNames: l_quantity - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(l_quantity) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: lineitem - Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (l_shipmode = 'AIR') (type: boolean) - Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4519,7 +4419,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4540,7 +4440,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4553,7 +4453,47 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 15 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: l_partkey is not null (type: boolean) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_partkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 16 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_quantity (type: double) + outputColumnNames: l_quantity + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: avg(l_quantity) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4570,7 +4510,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_quantity (type: double) + outputColumnNames: l_quantity + Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: avg(l_quantity) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4588,7 +4548,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4601,7 +4561,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4619,7 +4579,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4636,7 +4596,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col4 (type: int) Statistics: Num rows: 5 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col3 (type: int) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4658,7 +4618,47 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 5 + Map Operator Tree: + TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 17 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Map Operator Tree: + TableScan + alias: lineitem + Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (l_shipmode = 'AIR') (type: boolean) + Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4673,7 +4673,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: double), _col4 (type: int) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4690,7 +4690,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: double) Statistics: Num rows: 14 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col4 (type: int), _col5 (type: bigint), _col6 (type: bigint) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4725,7 +4725,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 8' is a cross product +Warning: Shuffle Join MERGEJOIN[100][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 8' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4765,21 +4765,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Reducer 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -4795,7 +4795,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -4815,74 +4815,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4905,7 +4838,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4931,7 +4864,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4944,7 +4877,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4965,7 +4914,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4978,7 +4927,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4991,7 +4957,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5014,7 +4980,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5032,7 +4998,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: bigint) Statistics: Num rows: 1 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5050,7 +5016,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5071,7 +5054,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5084,7 +5067,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5419,7 +5419,7 @@ POSTHOOK: Input: default@src 431 val_431 3 430 val_430 3 417 val_417 3 -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: explain select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_name IN (select p_name from part p where part.p_type <> '1') @@ -5433,13 +5433,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5456,42 +5456,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_type <> '1') (type: boolean) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5509,7 +5474,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + Vertex 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5530,7 +5510,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5543,7 +5523,27 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (p_type <> '1') (type: boolean) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5562,7 +5562,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 4' is a cross product PREHOOK: query: select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY PREHOOK: Input: default@part diff --git ql/src/test/results/clientpositive/llap/subquery_notin.q.out ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 48fe336..064dae6 100644 --- ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from src @@ -24,13 +24,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -45,46 +45,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(key) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -101,7 +62,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -126,7 +87,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(key) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -138,7 +118,27 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -163,7 +163,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where src.key not in ( select key from src s1 where s1.key > '2') @@ -324,21 +324,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -355,55 +355,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: p_name (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -417,24 +369,7 @@ STAGE PLANS: value expressions: p_name (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Select Operator @@ -475,7 +410,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -496,7 +431,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -514,7 +449,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -531,7 +466,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -544,7 +496,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -557,7 +526,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -574,7 +543,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 6214 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -599,7 +568,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: p_name (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -640,7 +623,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -667,7 +650,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -682,7 +665,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -740,7 +740,7 @@ Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[49][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select p_name, p_size from @@ -768,16 +768,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -792,33 +792,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -836,7 +810,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -853,7 +827,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col1) (type: double) Statistics: Num rows: 26 Data size: 3666 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -878,7 +852,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -922,7 +909,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -939,7 +926,20 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -983,7 +983,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1008,7 +1008,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in @@ -1080,27 +1080,27 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 22 <- Reducer 21 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) - Reducer 24 <- Map 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -1117,7 +1117,20 @@ STAGE PLANS: value expressions: _col0 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: part @@ -1130,114 +1143,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 23 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_mfgr (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1278,7 +1184,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1305,7 +1211,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1326,7 +1232,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1339,7 +1262,20 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1380,7 +1316,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 19 + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1407,7 +1343,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1431,7 +1367,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 8 Data size: 1848 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: int) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1450,7 +1386,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1468,7 +1404,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 22 + Vertex 22 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1485,7 +1421,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 24 + Vertex 23 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1498,7 +1451,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1511,7 +1481,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1528,7 +1498,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: int) Statistics: Num rows: 8 Data size: 1912 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col6 (type: bigint), _col7 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1553,7 +1523,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: p_mfgr (type: string), p_size (type: int) + sort order: ++ + Map-reduce partition columns: p_mfgr (type: string) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1593,7 +1576,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 8 Data size: 784 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1614,7 +1597,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1634,6 +1617,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_mfgr (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -1679,7 +1679,7 @@ Manufacturer#5 almond antique medium spring khaki 6 Manufacturer#5 almond azure blanched chiffon midnight 23 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 -Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select li.l_partkey, count(*) from lineitem li where li.l_linenumber = 1 and @@ -1712,7 +1712,7 @@ POSTHOOK: Input: default@lineitem 139636 1 175839 1 182052 1 -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where not src.key in ( select key from src s1 where s1.key > '2') @@ -1874,7 +1874,7 @@ POSTHOOK: Input: default@t1_v POSTHOOK: Output: database:default POSTHOOK: Output: default@T2_v POSTHOOK: Lineage: T2_v.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) @@ -1892,13 +1892,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1918,7 +1918,49 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4 + Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((_col1 = 0) or (_col4 is null and _col0 is not null and (_col2 >= _col1))) (type: boolean) + Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -1943,7 +1985,19 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 6 Map Operator Tree: TableScan alias: src @@ -1969,61 +2023,7 @@ STAGE PLANS: Statistics: Num rows: 35 Data size: 6440 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 17098 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4 - Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((_col1 = 0) or (_col4 is null and _col0 is not null and (_col2 >= _col1))) (type: boolean) - Statistics: Num rows: 166 Data size: 17762 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2048,7 +2048,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) PREHOOK: type: QUERY @@ -2076,19 +2076,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2108,92 +2108,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2218,7 +2133,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 6 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2236,7 +2151,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2253,7 +2168,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2266,7 +2198,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2279,7 +2228,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2296,7 +2245,7 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2321,7 +2270,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2348,7 +2314,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2363,7 +2329,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2376,6 +2359,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -2414,7 +2414,7 @@ POSTHOOK: Input: default@part 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de 144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) @@ -2428,15 +2428,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2451,53 +2451,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2514,7 +2468,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2539,29 +2493,52 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(_col1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2573,7 +2550,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2596,7 +2596,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2621,7 +2621,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2646,7 +2646,7 @@ POSTHOOK: Input: default@part 48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) @@ -2660,15 +2660,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -2683,53 +2683,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_partkey (type: int) - outputColumnNames: p_type, p_partkey - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_partkey (type: int) - outputColumnNames: p_type, p_partkey - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2746,7 +2700,7 @@ STAGE PLANS: Map-reduce partition columns: (_col0 * _col5) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -2771,7 +2725,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_partkey (type: int) + outputColumnNames: p_type, p_partkey + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_partkey) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2793,7 +2770,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2805,7 +2782,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_partkey (type: int) + outputColumnNames: p_type, p_partkey + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_partkey) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2828,7 +2828,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2853,7 +2853,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2900,20 +2900,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -2930,58 +2930,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 + Vertex 10 Map Operator Tree: TableScan alias: part @@ -2998,24 +2947,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3036,7 +2968,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3054,7 +2986,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3071,7 +3003,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3084,7 +3033,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3098,7 +3064,7 @@ STAGE PLANS: Map-reduce partition columns: (_col0 + 100) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3115,7 +3081,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 26 Data size: 3666 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3140,7 +3106,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3155,17 +3121,34 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string), _col0 (type: int) outputColumnNames: _col2, _col0 @@ -3182,7 +3165,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3197,7 +3180,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3226,7 +3226,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### 26 -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) @@ -3240,15 +3240,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3263,53 +3263,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_retailprice (type: double) - outputColumnNames: p_type, p_retailprice - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_retailprice) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_retailprice (type: double) - outputColumnNames: p_type, p_retailprice - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_retailprice) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3326,7 +3280,7 @@ STAGE PLANS: Map-reduce partition columns: floor(_col7) (type: bigint) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3351,7 +3305,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_retailprice (type: double) + outputColumnNames: p_type, p_retailprice + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_retailprice) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3373,7 +3350,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3385,7 +3362,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_retailprice (type: double) + outputColumnNames: p_type, p_retailprice + Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_retailprice) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3408,7 +3408,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3433,7 +3433,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[35][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3456,19 +3456,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3485,92 +3485,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3591,7 +3506,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 1 Data size: 125 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3609,7 +3524,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3626,7 +3541,24 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3639,7 +3571,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3652,7 +3601,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3669,7 +3618,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col5 (type: int), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3694,7 +3643,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3721,7 +3687,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3740,7 +3706,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col0 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3753,6 +3736,23 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -3805,19 +3805,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -3834,92 +3834,7 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_partkey (type: int), p_size (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: p - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col2 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3940,7 +3855,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int) Statistics: Num rows: 1 Data size: 129 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3958,7 +3873,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3975,7 +3890,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col4 (type: string), _col2 (type: int) Statistics: Num rows: 1 Data size: 133 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3988,7 +3920,24 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4001,7 +3950,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4018,7 +3967,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col5 (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4043,7 +3992,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4070,7 +4036,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4085,7 +4051,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_partkey (type: int), p_size (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4098,6 +4081,23 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4126,19 +4126,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4155,7 +4155,67 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (UDFToDouble(_col1) + 2.0) (type: double), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: double), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) + Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: double), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: double), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: boolean) + Vertex 12 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col3 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col3 (type: string), _col1 (type: string) + Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) + Vertex 13 Map Operator Tree: TableScan alias: part @@ -4172,148 +4232,37 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 14 Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 15 Map Operator Tree: TableScan alias: part - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: p_brand (type: string) + keys: p_type (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (UDFToDouble(_col1) + 2.0) (type: double), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: double), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: double), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: double) - 1 UDFToDouble(_col0) (type: double) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col3 (type: string), _col1 (type: string) - Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) - Reducer 14 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4327,7 +4276,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4344,7 +4293,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: string) Statistics: Num rows: 26 Data size: 8658 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4369,7 +4318,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4396,7 +4362,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4411,7 +4377,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4424,6 +4407,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4452,19 +4452,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -4481,100 +4481,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4595,7 +4502,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 9 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4613,7 +4520,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 9 Data size: 1008 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4630,7 +4537,28 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: int) Statistics: Num rows: 4 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4643,7 +4571,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4656,7 +4601,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4673,7 +4618,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col2 (type: int) Statistics: Num rows: 26 Data size: 6370 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4698,7 +4643,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4725,7 +4687,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4740,7 +4702,28 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (p_size + 1) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4753,6 +4736,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 9 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -4794,7 +4794,7 @@ almond azure aquamarine papaya violet almond antique medium spring khaki almond aquamarine sandy cyan gainsboro almond antique olive coral navajo -Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY POSTHOOK: query: explain select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) @@ -4808,17 +4808,17 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -4833,7 +4833,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s1 @@ -4857,44 +4857,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(key) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4920,7 +4883,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4933,7 +4896,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4950,7 +4913,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -4980,7 +4943,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4998,7 +4961,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5016,7 +4979,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(key) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5028,7 +5011,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5053,7 +5053,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[56][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -5106,23 +5106,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE) - Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 18 <- Map 17 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 20 <- Map 19 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 21 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -5139,82 +5139,20 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 - Map Operator Tree: - TableScan - alias: sc - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 17 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 19 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: '90' (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Vertex 10 Execution mode: llap - LLAP IO: no inputs - Map 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 11 Map Operator Tree: TableScan alias: sc @@ -5230,38 +5168,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + LLAP IO: no inputs + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5286,7 +5194,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5304,7 +5212,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 14 + Vertex 14 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5321,7 +5229,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: string) Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5334,7 +5259,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 18 + Vertex 17 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 18 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5347,7 +5289,31 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 19 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: '90' (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5364,7 +5330,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5390,7 +5356,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5403,7 +5369,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5433,7 +5399,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5451,7 +5417,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: bigint) Statistics: Num rows: 103 Data size: 9785 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5469,7 +5435,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: sc + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5496,7 +5479,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 21115 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5511,6 +5494,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 21115 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -5557,7 +5557,7 @@ POSTHOOK: Input: default@src 431 3 430 3 417 3 -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand @@ -5571,16 +5571,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5595,53 +5595,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5659,7 +5613,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5676,7 +5630,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5699,7 +5653,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -5713,7 +5667,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5735,7 +5712,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5747,7 +5724,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5777,7 +5777,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5802,7 +5802,7 @@ POSTHOOK: Input: default@part 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 @@ -5816,16 +5816,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: part @@ -5840,53 +5840,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5904,7 +5858,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5921,7 +5875,7 @@ STAGE PLANS: Map-reduce partition columns: (_col5 - 1) (type: int) Statistics: Num rows: 26 Data size: 16510 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5945,24 +5899,47 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Reducer 4 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 4 + Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 4 - Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 2476 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5984,7 +5961,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5996,7 +5973,30 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: p_type, p_size + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(p_size) + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6026,7 +6026,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -6039,7 +6039,7 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY POSTHOOK: query: explain select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) @@ -6053,17 +6053,21 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 11 <- Union 9 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Map 8 <- Union 9 (CONTAINS) - Reducer 10 <- Union 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE) + Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Union 9 (CONTAINS) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 6 (PARTITION_EDGE) + Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 5 (PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) + Vertex 8 <- Union 9 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 5 + Vertex: Union 5 + Union 9 + Vertex: Union 9 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -6078,7 +6082,25 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 + Vertex 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) + Vertex 11 Map Operator Tree: TableScan alias: part @@ -6099,7 +6121,49 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col5 + Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) (type: boolean) + Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: part @@ -6119,7 +6183,19 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Vertex 7 Map Operator Tree: TableScan alias: part @@ -6139,7 +6215,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: part @@ -6160,82 +6236,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col5 - Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((_col2 = 0) or (_col5 is null and _col0 is not null and (_col3 >= _col2))) (type: boolean) - Statistics: Num rows: 500 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Union 5 - Vertex: Union 5 - Union 9 - Vertex: Union 9 Stage: Stage-0 Fetch Operator @@ -6243,7 +6243,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -6767,20 +6767,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: e @@ -6796,93 +6796,25 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 2496 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Execution mode: llap - LLAP IO: no inputs - Map 10 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: e - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_brand (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + LLAP IO: no inputs + Vertex 10 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6903,7 +6835,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6921,7 +6853,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 13 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6938,7 +6870,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col3 (type: int) Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6951,7 +6900,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -6965,7 +6931,7 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble((_col0 + 100)) (type: double) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -6982,7 +6948,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7007,7 +6973,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7022,7 +6988,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7049,7 +7032,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7064,7 +7047,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: e + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_brand (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7124,7 +7124,7 @@ POSTHOOK: query: INSERT INTO t2 VALUES (null), (2), (100) POSTHOOK: type: QUERY POSTHOOK: Output: default@t2 POSTHOOK: Lineage: t2.c1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) PREHOOK: type: QUERY POSTHOOK: query: explain SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) @@ -7138,13 +7138,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7159,44 +7159,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: c1 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(c1) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7213,7 +7176,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 381 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7238,7 +7201,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: c1 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(c1) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7250,7 +7233,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7275,7 +7275,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: SELECT c1 FROM t1 WHERE c1 NOT IN (SELECT c1 FROM t2) PREHOOK: type: QUERY PREHOOK: Input: default@t1 @@ -7299,19 +7299,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7328,92 +7328,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7434,7 +7349,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7452,7 +7367,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: char(100)), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7469,7 +7384,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: char(100)) Statistics: Num rows: 2 Data size: 171 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c2 (type: char(100)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(100)) + sort order: + + Map-reduce partition columns: _col0 (type: char(100)) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7483,7 +7415,24 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: char(100)) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7496,7 +7445,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7513,7 +7462,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) Statistics: Num rows: 4 Data size: 344 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7538,7 +7487,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7561,7 +7527,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7576,7 +7542,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: c2 (type: char(100)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(100)) + sort order: + + Map-reduce partition columns: _col0 (type: char(100)) + Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7590,6 +7573,23 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 2 Data size: 156 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: char(100)) + Vertex 9 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -7670,19 +7670,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -7699,92 +7699,7 @@ STAGE PLANS: value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: b (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7805,7 +7720,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7823,7 +7738,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7840,7 +7755,24 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col3 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7853,7 +7785,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: b (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7866,7 +7815,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7883,7 +7832,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7908,7 +7857,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7935,7 +7901,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7950,7 +7916,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -7963,6 +7946,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8029,133 +8029,48 @@ POSTHOOK: Output: default@fixob POSTHOOK: Lineage: fixob.i EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: fixob.j EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col2, type:string, comment:), ] PREHOOK: query: explain select * from fixOb where j NOT IN (select i from t7 where t7.j=fixOb.j) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from fixOb where j NOT IN (select i from t7 where t7.j=fixOb.j) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from fixOb where j NOT IN (select i from t7 where t7.j=fixOb.j) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +#### A masked pattern was here #### + Vertices: + Vertex 1 Map Operator Tree: TableScan alias: fixob Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: fixob - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t7 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8176,7 +8091,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8194,7 +8109,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8211,7 +8126,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8224,7 +8156,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8237,7 +8186,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8254,7 +8203,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8279,7 +8228,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t7 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8306,7 +8272,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8321,7 +8287,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: fixob + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 9 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8334,6 +8317,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t7 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8397,108 +8397,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 + Vertex 1 Map Operator Tree: TableScan alias: t Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 @@ -8511,7 +8426,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8532,7 +8447,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8550,7 +8465,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8567,7 +8482,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8580,7 +8512,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8593,7 +8542,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8610,7 +8559,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8635,7 +8584,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8662,7 +8628,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8677,7 +8643,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8690,6 +8673,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -8721,108 +8721,23 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 13 + Vertex 1 Map Operator Tree: TableScan alias: t Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: i (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: t - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 @@ -8835,7 +8750,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8856,7 +8771,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8874,7 +8789,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8891,7 +8806,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8904,7 +8836,24 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -8917,7 +8866,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8934,7 +8883,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8959,7 +8908,24 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -8986,7 +8952,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 11 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9001,7 +8967,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9014,6 +8997,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Vertex 9 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -9030,7 +9030,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@t #### A masked pattern was here #### 7 -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select t.i from t where t.j NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY POSTHOOK: query: explain select t.i from t where t.j NOT IN (select t1.i from t t1 ) @@ -9044,13 +9044,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -9065,44 +9065,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(i) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: i (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9119,7 +9082,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 3 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: bigint), _col3 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9144,7 +9107,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(i) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9156,7 +9139,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9181,7 +9181,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select t.i from t where t.j NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY PREHOOK: Input: default@t @@ -9192,7 +9192,7 @@ POSTHOOK: Input: default@t #### A masked pattern was here #### 1 4 -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select t.i from t where t.i NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY POSTHOOK: query: explain select t.i from t where t.i NOT IN (select t1.i from t t1 ) @@ -9206,13 +9206,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t @@ -9227,44 +9227,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(), count(i) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: i (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9281,7 +9244,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9306,7 +9269,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(), count(i) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9318,7 +9301,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9343,7 +9343,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select t.i from t where t.i NOT IN (select t1.i from t t1 ) PREHOOK: type: QUERY PREHOOK: Input: default@t @@ -9356,8 +9356,8 @@ PREHOOK: query: drop table t1 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t1 POSTHOOK: type: DROPTABLE -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 10' is a cross product PREHOOK: query: explain select * from src b where b.key not in @@ -9383,19 +9383,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Reducer 14 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (SIMPLE_EDGE) - Reducer 16 <- Map 15 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Vertex 10 <- Vertex 14 (PARTITION_EDGE), Vertex 9 (PARTITION_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) + Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) + Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE), Vertex 8 (PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) + Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -9412,94 +9412,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 15 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9527,7 +9440,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1656 Data size: 294768 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9545,7 +9458,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1656 Data size: 301392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9562,7 +9475,24 @@ STAGE PLANS: Map-reduce partition columns: _col3 (type: string), _col1 (type: string) Statistics: Num rows: 1656 Data size: 301392 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) - Reducer 14 + Vertex 13 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 14 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9574,7 +9504,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 16 + Vertex 15 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 16 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9587,7 +9534,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9604,7 +9551,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 97000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: bigint), _col4 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9629,7 +9576,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -9659,7 +9624,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 7704 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9674,7 +9639,24 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 72 Data size: 7704 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -9686,6 +9668,24 @@ STAGE PLANS: sort order: Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) + Vertex 9 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -9693,8 +9693,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[70][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[72][tables = [$hdt$_2, $hdt$_3]] in Stage 'Vertex 10' is a cross product PREHOOK: query: select * from src b where b.key not in diff --git ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out index 7d9d77c..b895a5b 100644 --- ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out +++ ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out @@ -6,8 +6,8 @@ POSTHOOK: query: CREATE TABLE table_7 (int_col INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@table_7 -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain SELECT (t1.int_col) * (t1.int_col) AS int_col @@ -55,13 +55,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 6 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE) + Vertex 5 <- Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: table_7 @@ -73,50 +73,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: table_7 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count(), count(false) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: table_7 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - keys: false (type: boolean) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -131,7 +88,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -156,7 +113,28 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count(), count(false) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -175,7 +153,29 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: table_7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + keys: false (type: boolean) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/subquery_views.q.out ql/src/test/results/clientpositive/llap/subquery_views.q.out index d96a5a4..25a45e2 100644 --- ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -124,31 +124,31 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) - Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) - Reducer 15 <- Map 14 (SIMPLE_EDGE) - Reducer 17 <- Map 16 (SIMPLE_EDGE) - Reducer 19 <- Map 18 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 20 <- Reducer 19 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) - Reducer 21 <- Reducer 20 (SIMPLE_EDGE) - Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) - Reducer 24 <- Reducer 23 (SIMPLE_EDGE) - Reducer 26 <- Map 25 (SIMPLE_EDGE) - Reducer 28 <- Map 27 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) - Reducer 29 <- Reducer 28 (SIMPLE_EDGE) - Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) - Reducer 32 <- Map 31 (SIMPLE_EDGE) - Reducer 34 <- Map 33 (SIMPLE_EDGE) - Reducer 4 <- Reducer 21 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) + Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) + Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) + Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) + Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) + Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) + Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) + Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) + Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) + Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) + Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) + Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) + Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE) + Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -169,7 +169,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: a @@ -188,199 +188,7 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 14 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 16 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 18 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < '11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 22 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 25 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 27 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 31 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 33 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (value > 'val_11') (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - properties: - insideView TRUE - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 11 + Vertex 11 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -401,7 +209,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 12 + Vertex 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -419,7 +227,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: boolean) - Reducer 13 + Vertex 13 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -436,7 +244,26 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 15 + Vertex 14 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -449,7 +276,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 17 + Vertex 16 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 17 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -462,7 +308,28 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 19 + Vertex 18 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < '11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 19 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -479,7 +346,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col0 (type: string) Statistics: Num rows: 166 Data size: 32204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint), _col5 (type: bigint) - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -496,7 +363,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col0 (type: string) Statistics: Num rows: 166 Data size: 32204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col4 (type: bigint), _col5 (type: bigint) - Reducer 20 + Vertex 20 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -524,7 +391,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 21 + Vertex 21 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -537,7 +404,26 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 23 + Vertex 22 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 23 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -564,7 +450,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 24 + Vertex 24 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -579,7 +465,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 26 + Vertex 25 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 26 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -592,7 +497,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 28 + Vertex 27 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 28 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -613,7 +537,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 29 + Vertex 29 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -631,7 +555,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -655,7 +579,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 30 + Vertex 30 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -672,7 +596,26 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) - Reducer 32 + Vertex 31 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 32 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -685,7 +628,26 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 34 + Vertex 33 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 34 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -698,7 +660,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -716,7 +678,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (value > 'val_11') (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -743,7 +724,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -758,7 +739,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 9 + Vertex 8 + Map Operator Tree: + TableScan + alias: b + properties: + insideView TRUE + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out index 48ea4ce..b7a6368 100644 --- ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out +++ ql/src/test/results/clientpositive/llap/table_access_keys_stats.q.out @@ -496,7 +496,7 @@ Keys:val 13.0 1 17.0 1 46.0 1 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 4' is a cross product PREHOOK: query: SELECT * FROM ( diff --git ql/src/test/results/clientpositive/llap/temp_table.q.out ql/src/test/results/clientpositive/llap/temp_table.q.out index e97b3dd..a14b118 100644 --- ql/src/test/results/clientpositive/llap/temp_table.q.out +++ ql/src/test/results/clientpositive/llap/temp_table.q.out @@ -14,7 +14,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -86,7 +86,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -171,10 +171,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: foo @@ -191,7 +191,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -246,12 +246,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: foo @@ -268,24 +270,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: bar - Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -302,8 +287,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 + Vertex 4 + Map Operator Tree: + TableScan + alias: bar + Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 253 Data size: 2703 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out index c83da8b..f9d36da 100644 --- ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out +++ ql/src/test/results/clientpositive/llap/tez_bmj_schema_evolution.q.out @@ -79,11 +79,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (CUSTOM_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (CO_PARTITION_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test @@ -103,7 +103,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -112,7 +112,21 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: some inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: test1 @@ -131,20 +145,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1100 Data size: 66343 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_dml.q.out ql/src/test/results/clientpositive/llap/tez_dml.q.out index 786929e..dcb9aca 100644 --- ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -14,11 +14,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -41,7 +41,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -55,7 +55,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 214 Data size: 21186 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -444,7 +444,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tmp_src @@ -865,7 +865,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src diff --git ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out index 33350a7..d20b618 100644 --- ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -47,27 +47,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -83,7 +63,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 2166 Data size: 1342920 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -97,6 +77,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -178,26 +178,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -216,7 +197,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -231,6 +212,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -282,12 +282,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -307,26 +307,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -349,7 +330,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -363,7 +344,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -377,6 +358,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -434,11 +434,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -458,27 +458,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -489,7 +469,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Reduce Output Operator @@ -497,7 +477,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -511,6 +491,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -592,26 +592,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -621,7 +602,7 @@ STAGE PLANS: 0 KEY.reducesinkkey0 (type: int) 1 KEY.reducesinkkey0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -633,7 +614,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -648,6 +629,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -699,12 +699,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -724,26 +724,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Map Join Operator @@ -754,7 +735,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -769,7 +750,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -783,7 +764,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 750 Data size: 161305 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -797,6 +778,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out index f434a1e..a92ef33 100644 --- ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_2.q.out @@ -31,11 +31,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -55,7 +55,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -74,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -93,38 +125,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -222,11 +222,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -246,7 +246,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -265,7 +297,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -284,38 +316,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -413,11 +413,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -437,7 +437,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -456,7 +488,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -475,38 +507,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out index 2918e68..3bf7561 100644 --- ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out +++ ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out @@ -19,11 +19,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -42,23 +42,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) - sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -74,7 +58,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -98,6 +82,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToInteger(_col0) (type: int) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -126,10 +126,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -148,23 +148,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToInteger(_col0) (type: int) - sort order: + - Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -185,7 +169,7 @@ STAGE PLANS: 1 UDFToInteger(_col0) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 2 Data size: 620 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 @@ -197,6 +181,22 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToInteger(_col0) (type: int) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_join_hash.q.out ql/src/test/results/clientpositive/llap/tez_join_hash.q.out index ef4556a..75dd571 100644 --- ql/src/test/results/clientpositive/llap/tez_join_hash.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_hash.q.out @@ -31,11 +31,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -54,26 +54,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: orc_src - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -92,7 +73,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -107,6 +88,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: orc_src + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -146,91 +146,35 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Map 8 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: y - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: x - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: z - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 8 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -240,18 +184,18 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1, _col2 - Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19512 Data size: 3473136 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -261,20 +205,20 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 50261 Data size: 8946458 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 50261 Data size: 8946458 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col2 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -282,20 +226,76 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 4674180 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25130 Data size: 2387350 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 21512 Data size: 3829136 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 8 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out index a7ec895..5baa46c 100644 --- ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out @@ -160,10 +160,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -245,7 +245,7 @@ STAGE PLANS: name: default.ct_events_clean Truncated Path -> Alias: /ct_events_clean [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -262,13 +262,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 4 Data size: 19978 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1144,10 +1144,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -1229,7 +1229,7 @@ STAGE PLANS: name: default.ct_events_clean Truncated Path -> Alias: /ct_events_clean [a] - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1246,13 +1246,13 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 2 + Estimated key counts: Vertex 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Position of Big Table: 1 Statistics: Num rows: 4 Data size: 19978 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/llap/tez_join_tests.q.out ql/src/test/results/clientpositive/llap/tez_join_tests.q.out index 4fa5854..fde7ff8 100644 --- ql/src/test/results/clientpositive/llap/tez_join_tests.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_tests.q.out @@ -13,13 +13,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -36,40 +36,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +52,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,7 +65,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -114,7 +81,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -128,6 +95,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out index b32e990..77602e9 100644 --- ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out +++ ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out @@ -13,13 +13,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: b @@ -36,40 +36,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -85,7 +52,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -98,7 +65,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 60 Data size: 10680 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -114,7 +81,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -128,6 +95,39 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_nway_join.q.out ql/src/test/results/clientpositive/llap/tez_nway_join.q.out index cee5b9c..20a5970 100644 --- ql/src/test/results/clientpositive/llap/tez_nway_join.q.out +++ ql/src/test/results/clientpositive/llap/tez_nway_join.q.out @@ -41,11 +41,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -66,8 +66,8 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) input vertices: - 1 Map 3 - 2 Map 4 + 1 Vertex 3 + 2 Vertex 4 Statistics: Num rows: 101 Data size: 404 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -80,7 +80,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -99,7 +114,7 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -118,21 +133,6 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -153,11 +153,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -177,7 +177,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 50 Data size: 202 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -186,7 +186,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -199,7 +199,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -218,7 +233,7 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -237,21 +252,6 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -294,11 +294,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -314,39 +314,7 @@ STAGE PLANS: Statistics: Num rows: 46 Data size: 184 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: c - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -367,7 +335,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -382,6 +350,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -402,11 +402,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -423,7 +423,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 50 Data size: 202 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -432,7 +432,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -445,7 +445,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -461,7 +476,7 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: c @@ -477,21 +492,6 @@ STAGE PLANS: Statistics: Num rows: 45 Data size: 183 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_self_join.q.out ql/src/test/results/clientpositive/llap/tez_self_join.q.out index 8ba9761..470a14a 100644 --- ql/src/test/results/clientpositive/llap/tez_self_join.q.out +++ ql/src/test/results/clientpositive/llap/tez_self_join.q.out @@ -38,7 +38,7 @@ POSTHOOK: query: insert into table tez_self_join2 values(1),(2),(3) POSTHOOK: type: QUERY POSTHOOK: Output: default@tez_self_join2 POSTHOOK: Lineage: tez_self_join2.id1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: explain select s.id2, s.id3 from @@ -70,11 +70,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: self1 @@ -92,41 +92,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: self2 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ('ab' = id3) (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: tez_self_join2 - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: id1 is not null (type: boolean) - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: id1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -147,7 +113,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -169,6 +135,40 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: self2 + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ('ab' = id3) (type: boolean) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: tez_self_join2 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: id1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -176,7 +176,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product PREHOOK: query: select s.id2, s.id3 from ( diff --git ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out index d9d2db4..1bb66c6 100644 --- ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out +++ ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out @@ -131,10 +131,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -174,7 +174,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -224,7 +224,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -533,10 +533,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -589,7 +589,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -639,10 +639,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -695,7 +695,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -745,10 +745,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -788,7 +788,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -834,10 +834,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: s3 @@ -877,7 +877,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/tez_smb_main.q.out ql/src/test/results/clientpositive/llap/tez_smb_main.q.out index b583bff..5a03f39 100644 --- ql/src/test/results/clientpositive/llap/tez_smb_main.q.out +++ ql/src/test/results/clientpositive/llap/tez_smb_main.q.out @@ -13,10 +13,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -36,7 +36,25 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -56,24 +74,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 60 Data size: 21180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -278,7 +278,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -297,7 +297,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -310,7 +310,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -389,7 +389,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -408,7 +408,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -421,7 +421,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -500,7 +500,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -519,7 +519,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -532,7 +532,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -626,7 +626,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -638,7 +638,22 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Execution mode: llap - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: c @@ -657,21 +672,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -732,26 +732,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -770,7 +751,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -785,6 +766,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -871,7 +871,41 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 508 Data size: 5388 Basic stats: COMPLETE Column stats: NONE Execution mode: llap - Map 6 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Statistics: Num rows: 558 Data size: 5926 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: s2 @@ -890,7 +924,7 @@ STAGE PLANS: Statistics: Num rows: 508 Data size: 5388 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: b @@ -909,42 +943,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Statistics: Num rows: 558 Data size: 5926 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -988,7 +986,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1007,7 +1005,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col1 (type: string) input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1020,7 +1018,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1094,7 +1092,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -1114,7 +1112,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1123,7 +1121,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1136,7 +1134,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: c @@ -1155,21 +1168,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1240,7 +1238,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 266 Data size: 4952 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -1249,7 +1247,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 558 Data size: 10399 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1262,7 +1260,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -1281,7 +1294,7 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s2 @@ -1300,7 +1313,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 558 Data size: 10399 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1313,7 +1326,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: b @@ -1337,23 +1350,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1411,7 +1407,19 @@ STAGE PLANS: Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -1429,19 +1437,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1455,7 +1451,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 10243 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1466,7 +1462,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/tez_union.q.out ql/src/test/results/clientpositive/llap/tez_union.q.out index 395961e..1903c8a 100644 --- ql/src/test/results/clientpositive/llap/tez_union.q.out +++ ql/src/test/results/clientpositive/llap/tez_union.q.out @@ -17,11 +17,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -41,7 +43,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -52,7 +54,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s3 @@ -71,7 +73,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -89,8 +91,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -171,15 +171,19 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Map 6 <- Union 7 (CONTAINS) - Map 8 <- Union 7 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) + Vertex 6 <- Union 7 (CONTAINS) + Vertex 8 <- Union 7 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 7 + Vertex: Union 7 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -198,7 +202,41 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 4878 Data size: 39024 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: src @@ -217,7 +255,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: src @@ -236,7 +274,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: src @@ -255,44 +293,6 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 4878 Data size: 39024 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 - Union 7 - Vertex: Union 7 Stage: Stage-0 Fetch Operator @@ -351,11 +351,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) - Map 4 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (BROADCAST_EDGE) + Vertex 4 <- Union 3 (CONTAINS), Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -379,7 +381,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: src @@ -399,7 +401,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2439 Data size: 424386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -410,7 +412,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: src @@ -430,7 +432,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 2439 Data size: 424386 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -441,8 +443,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -529,13 +529,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 10 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 5 <- Map 10 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 7 <- Map 10 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 4 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 6 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) + Vertex 7 <- Union 2 (CONTAINS), Vertex 10 (BROADCAST_EDGE), Vertex 8 (BROADCAST_EDGE), Vertex 9 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -555,7 +557,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -567,8 +569,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -581,7 +583,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: s8 @@ -610,7 +612,21 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -629,7 +645,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s4 @@ -649,7 +665,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 6 + 1 Vertex 6 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -661,8 +677,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -675,7 +691,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: s5 @@ -694,7 +710,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: s6 @@ -714,7 +730,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 8 + 1 Vertex 8 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -726,8 +742,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 9 - 2 Map 10 + 1 Vertex 9 + 2 Vertex 10 Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) @@ -740,7 +756,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: s7 @@ -759,7 +775,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 9 + Vertex 9 Map Operator Tree: TableScan alias: s1 @@ -788,22 +804,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 21754 Data size: 5677794 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -971,11 +971,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 3 (BROADCAST_EDGE) + Vertex 4 <- Union 2 (CONTAINS), Vertex 5 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -995,7 +997,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1006,7 +1008,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s3 @@ -1025,7 +1027,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s4 @@ -1045,7 +1047,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1219 Data size: 106053 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1056,7 +1058,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s5 @@ -1075,8 +1077,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1164,11 +1164,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) - Map 3 <- Map 4 (BROADCAST_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) + Vertex 3 <- Union 2 (CONTAINS), Vertex 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1185,7 +1187,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2439 Data size: 868284 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1196,7 +1198,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1213,7 +1215,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 2439 Data size: 868284 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -1224,7 +1226,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s @@ -1247,8 +1249,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -1275,12 +1275,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (BROADCAST_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -1296,7 +1298,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -1312,7 +1314,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s @@ -1340,8 +1342,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_union2.q.out ql/src/test/results/clientpositive/llap/tez_union2.q.out index 51d45d2..e1a4bd0 100644 --- ql/src/test/results/clientpositive/llap/tez_union2.q.out +++ ql/src/test/results/clientpositive/llap/tez_union2.q.out @@ -41,14 +41,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 5 (CONTAINS) - Map 7 <- Union 5 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Union 2 (CONTAINS), Union 5 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 5 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS), Union 5 (SORT_PARTITION_EDGE) + Vertex 7 <- Union 5 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Union 5 + Vertex: Union 5 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -69,28 +73,22 @@ STAGE PLANS: Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 3 Execution mode: llap - LLAP IO: no inputs - Map 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -111,22 +109,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -157,10 +140,27 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 352 Data size: 62656 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 - Union 5 - Vertex: Union 5 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out index 2ca78d7..45f39d3 100644 --- ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out @@ -53,11 +53,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: dummy @@ -80,7 +82,7 @@ STAGE PLANS: name: default.partunion1 Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: dummy @@ -103,8 +105,6 @@ STAGE PLANS: name: default.partunion1 Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out index 22f24c9..b9b061a 100644 --- ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out @@ -147,16 +147,18 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Map 6 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 2 (CONTAINS) + Vertex 6 <- Union 2 (CONTAINS) + Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: x @@ -182,7 +184,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: v @@ -197,7 +199,50 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Execution mode: llap LLAP IO: all inputs - Map 5 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (_col1 <= '2014-09-02') (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: bigint), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: y @@ -223,7 +268,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: z @@ -249,7 +294,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: unknown - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -265,50 +310,7 @@ STAGE PLANS: value expressions: u (type: bigint) Execution mode: llap LLAP IO: unknown - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Filter Operator - predicate: (_col1 <= '2014-09-02') (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join0 to 1 - keys: - 0 _col0 (type: bigint) - 1 _col0 (type: bigint) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: bigint), _col0 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 8 + Vertex 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -330,7 +332,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Reducer 9 + Vertex 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -343,8 +345,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out index dc1e778..88c5e97 100644 --- ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out @@ -53,14 +53,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Map 7 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) + Vertex 7 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -78,73 +80,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s0 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -178,7 +114,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -195,7 +131,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -212,8 +148,72 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: s0 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -952,14 +952,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS), Vertex 5 (PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s0 @@ -992,58 +994,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1060,7 +1011,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1077,7 +1028,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1111,8 +1080,39 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -1847,14 +1847,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 7 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Union 2 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 2 (CONTAINS), Vertex 5 (PARTITION_EDGE) + Vertex 7 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s0 @@ -1887,58 +1889,7 @@ STAGE PLANS: Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1955,7 +1906,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1972,7 +1923,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2006,8 +1975,39 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE - Union 2 - Vertex: Union 2 + Vertex 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 140000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1001 Data size: 464464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -2734,13 +2734,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -2758,40 +2760,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(DISTINCT substr(_col1, 5)) - keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2825,7 +2794,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2842,7 +2811,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2859,8 +2828,39 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 70000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT substr(_col1, 5)) + keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 501 Data size: 232464 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection @@ -3581,13 +3581,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 6 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -3605,28 +3607,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3648,7 +3629,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3685,7 +3666,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 - Reducer 5 + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3702,8 +3683,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Union 3 - Vertex: Union 3 + Vertex 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 250 Data size: 68000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs Stage: Stage-3 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out index 5bb596a..20523cb 100644 --- ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out @@ -23,11 +23,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -47,27 +47,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -83,7 +63,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 2166 Data size: 1342920 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -97,6 +77,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -178,26 +178,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -216,7 +197,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -231,6 +212,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -282,12 +282,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -307,26 +307,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -349,7 +330,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -363,7 +344,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 615 Data size: 7380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -377,6 +358,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -434,11 +434,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -458,27 +458,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -489,7 +469,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Reduce Output Operator @@ -497,7 +477,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -511,6 +491,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -592,26 +592,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -621,7 +602,7 @@ STAGE PLANS: 0 KEY.reducesinkkey0 (type: int) 1 KEY.reducesinkkey0 (type: int) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -633,7 +614,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -648,6 +629,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -699,12 +699,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -724,26 +724,7 @@ STAGE PLANS: value expressions: _col0 (type: smallint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Map Join Operator @@ -754,7 +735,7 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Vertex 5 Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -769,7 +750,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1501 Data size: 322826 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -783,7 +764,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 750 Data size: 161305 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -797,6 +778,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1365 Data size: 293479 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out index 9c1b3af..e74ea96 100644 --- ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_2.q.out @@ -31,11 +31,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -55,7 +55,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -74,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -93,38 +125,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -222,11 +222,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -246,7 +246,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -265,7 +297,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -284,38 +316,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -413,11 +413,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -437,7 +437,39 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 UDFToInteger(_col1) (type: int) + 1 UDFToInteger(_col0) (type: int) + 2 (UDFToInteger(_col0) + 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) + sort order: +++ + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Vertex 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -456,7 +488,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: c @@ -475,38 +507,6 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 UDFToInteger(_col1) (type: int) - 1 UDFToInteger(_col0) (type: int) - 2 (UDFToInteger(_col0) + 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: smallint), _col0 (type: tinyint), _col2 (type: int) - sort order: +++ - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey0 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: string), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: timestamp), VALUE._col7 (type: boolean), VALUE._col8 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2710 Data size: 840100 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union2.q.out ql/src/test/results/clientpositive/llap/union2.q.out index 8b889af..fdf5bcd 100644 --- ql/src/test/results/clientpositive/llap/union2.q.out +++ ql/src/test/results/clientpositive/llap/union2.q.out @@ -15,12 +15,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -40,7 +42,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -60,23 +77,6 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union3.q.out ql/src/test/results/clientpositive/llap/union3.q.out index 4f07f85..4f20960 100644 --- ql/src/test/results/clientpositive/llap/union3.q.out +++ ql/src/test/results/clientpositive/llap/union3.q.out @@ -41,62 +41,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 10 <- Union 3 (CONTAINS), Vertex 9 (PARTITION_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) + Vertex 8 <- Union 3 (CONTAINS), Vertex 7 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Execution mode: llap - LLAP IO: no inputs - Map 9 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -112,7 +66,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 10 + Vertex 10 Execution mode: llap Reduce Operator Tree: Limit @@ -127,7 +81,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Limit @@ -142,7 +96,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Select Operator @@ -156,7 +110,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Limit @@ -171,7 +141,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs + Vertex 8 Execution mode: llap Reduce Operator Tree: Limit @@ -186,8 +172,22 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Union 3 - Vertex: Union 3 + Vertex 9 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union4.q.out ql/src/test/results/clientpositive/llap/union4.q.out index b9ca42d..501ab33 100644 --- ql/src/test/results/clientpositive/llap/union4.q.out +++ ql/src/test/results/clientpositive/llap/union4.q.out @@ -29,11 +29,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -51,25 +53,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -93,7 +77,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -117,8 +119,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/union5.q.out ql/src/test/results/clientpositive/llap/union5.q.out index c6a8712..61ecec4 100644 --- ql/src/test/results/clientpositive/llap/union5.q.out +++ ql/src/test/results/clientpositive/llap/union5.q.out @@ -17,12 +17,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Union 3 (CONTAINS), Vertex 5 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -42,27 +44,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(key) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -86,7 +68,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -102,7 +84,27 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(key) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -126,8 +128,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union6.q.out ql/src/test/results/clientpositive/llap/union6.q.out index dca14c1..498f0f0 100644 --- ql/src/test/results/clientpositive/llap/union6.q.out +++ ql/src/test/results/clientpositive/llap/union6.q.out @@ -29,11 +29,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 4 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -51,26 +53,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 26 Data size: 4647 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -90,8 +73,25 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable - Union 3 - Vertex: Union 3 + Vertex 4 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 4647 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Execution mode: llap + LLAP IO: no inputs Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/union7.q.out ql/src/test/results/clientpositive/llap/union7.q.out index 61a5ea4..2a0cc3b 100644 --- ql/src/test/results/clientpositive/llap/union7.q.out +++ ql/src/test/results/clientpositive/llap/union7.q.out @@ -17,12 +17,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 5 <- Union 3 (CONTAINS) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 4 <- Union 3 (SIMPLE_EDGE) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 4 <- Union 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -42,30 +44,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -89,7 +68,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -105,8 +84,29 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: no inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union8.q.out ql/src/test/results/clientpositive/llap/union8.q.out index 4f3be5f..04317b9 100644 --- ql/src/test/results/clientpositive/llap/union8.q.out +++ ql/src/test/results/clientpositive/llap/union8.q.out @@ -17,12 +17,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -40,7 +42,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: s2 @@ -58,7 +60,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: s3 @@ -76,8 +78,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: no inputs - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union9.q.out ql/src/test/results/clientpositive/llap/union9.q.out index 94e6c02..dfc400e 100644 --- ql/src/test/results/clientpositive/llap/union9.q.out +++ ql/src/test/results/clientpositive/llap/union9.q.out @@ -17,13 +17,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Map 5 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) + Vertex 5 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: s1 @@ -43,7 +45,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: s2 @@ -63,7 +80,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: s3 @@ -83,23 +100,6 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union_remove_26.q.out ql/src/test/results/clientpositive/llap/union_remove_26.q.out index 18b9aa5..72a383d 100644 --- ql/src/test/results/clientpositive/llap/union_remove_26.q.out +++ ql/src/test/results/clientpositive/llap/union_remove_26.q.out @@ -357,12 +357,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: inputtbl1 @@ -557,7 +557,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 3 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: inputtbl2 @@ -577,7 +592,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: inputtbl3 @@ -597,37 +627,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -720,7 +720,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: inputtbl2 @@ -740,7 +755,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: inputtbl3 @@ -760,37 +790,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), min(VALUE._col1), max(VALUE._col2) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -805,8 +805,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/union_stats.q.out ql/src/test/results/clientpositive/llap/union_stats.q.out index 1f3dc82..1a58a38 100644 --- ql/src/test/results/clientpositive/llap/union_stats.q.out +++ ql/src/test/results/clientpositive/llap/union_stats.q.out @@ -14,11 +14,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 3 <- Union 2 (CONTAINS) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -102,7 +104,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: src @@ -186,8 +188,6 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: /src [src] - Union 2 - Vertex: Union 2 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/union_top_level.q.out ql/src/test/results/clientpositive/llap/union_top_level.q.out index de23f48..bdbc193 100644 --- ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -21,12 +21,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -48,7 +50,28 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), 0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: src @@ -70,7 +93,28 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: src @@ -92,49 +136,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), 0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), 1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -155,8 +157,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 3 - Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -208,13 +208,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) - Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 4 (CONTAINS) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) + Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE) + Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) + Vertex 8 <- Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 4 + Vertex: Union 4 + Vertex 1 Map Operator Tree: TableScan alias: s2 @@ -234,7 +236,45 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Vertex 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 5 Map Operator Tree: TableScan alias: s1 @@ -253,7 +293,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: s2 @@ -273,26 +313,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 9 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 7 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -313,7 +334,7 @@ STAGE PLANS: Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string) - Reducer 3 + Vertex 8 Execution mode: llap Reduce Operator Tree: Select Operator @@ -330,46 +351,25 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: string) - Reducer 8 + Vertex 9 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 10 - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 + LLAP IO: no inputs Stage: Stage-0 Fetch Operator @@ -437,12 +437,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -464,51 +466,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -530,7 +488,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -552,7 +532,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -574,8 +576,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection @@ -670,12 +670,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -697,51 +699,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -763,7 +721,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -785,7 +765,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -807,8 +809,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection @@ -890,12 +890,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) - Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (PARTITION_EDGE) + Vertex 5 <- Union 3 (CONTAINS), Vertex 4 (PARTITION_EDGE) + Vertex 7 <- Union 3 (CONTAINS), Vertex 6 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Union 3 + Vertex: Union 3 + Vertex 1 Map Operator Tree: TableScan alias: src @@ -917,51 +919,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 3 - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string) - Execution mode: llap - LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -983,7 +941,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1005,7 +985,29 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Vertex 7 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1027,8 +1029,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_top - Union 3 - Vertex: Union 3 Stage: Stage-2 Dependency Collection diff --git ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out index 8482ed9..80516df 100644 --- ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out +++ ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out @@ -117,7 +117,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -185,7 +185,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -253,7 +253,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -321,7 +321,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -389,7 +389,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -457,7 +457,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: varchar_udf_1 @@ -515,7 +515,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -609,7 +609,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -667,7 +667,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -761,7 +761,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -821,10 +821,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: count_case_groupby @@ -847,7 +847,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -898,10 +898,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: count_case_groupby @@ -924,7 +924,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out index b093ded..b707c87 100644 --- ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out +++ ql/src/test/results/clientpositive/llap/vector_aggregate_9.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -139,7 +139,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out index fdd3d1b..fb36f7e 100644 --- ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out +++ ql/src/test/results/clientpositive/llap/vector_aggregate_without_gby.q.out @@ -38,26 +38,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 vectorized, llap - File Output Operator [FS_14] - Group By Operator [GBY_13] (rows=1 width=188) - Output:["_col0","_col1"],aggregations:["max(VALUE._col0)","max(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap - PARTITION_ONLY_SHUFFLE [RS_12] - Group By Operator [GBY_11] (rows=1 width=188) - Output:["_col0","_col1"],aggregations:["max(dt)","max(greg_dt)"] - Select Operator [SEL_10] (rows=3 width=102) - Output:["dt","greg_dt"] - Filter Operator [FIL_9] (rows=3 width=102) - predicate:(id = 5) - TableScan [TS_0] (rows=7 width=102) - default@testvec,testvec,Tbl:COMPLETE,Col:NONE,Output:["id","dt","greg_dt"] + Vertex 2 PREHOOK: query: select max(dt), max(greg_dt) from testvec where id=5 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out index bc37741..9001d17 100644 --- ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out @@ -218,68 +218,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 llap - File Output Operator [FS_32] - Select Operator [SEL_31] (rows=5 width=102) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_49] (rows=5 width=102) - Conds:RS_51._col0=RS_53._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 2 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_51] - PartitionCols:_col0 - Group By Operator [GBY_50] (rows=5 width=93) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=11 width=93) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Merge Join Operator [MERGEJOIN_45] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_42] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_41] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Reducer 6 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_53] - PartitionCols:_col0 - Group By Operator [GBY_52] (rows=5 width=93) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=11 width=93) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Merge Join Operator [MERGEJOIN_47] (rows=11 width=93) - Conds:SEL_16._col0=SEL_19._col0(Inner),Output:["_col0"] - <-Select Operator [SEL_19] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_44] (rows=10 width=93) - predicate:key is not null - TableScan [TS_17] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_16] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_43] (rows=10 width=93) - predicate:key is not null - TableScan [TS_14] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key"] + Vertex 3 PREHOOK: query: select src1.key, src1.cnt1, src2.cnt1 from ( @@ -992,30 +939,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Map 1 llap - File Output Operator [FS_11] - table:{"name:":"default.dest1"} - Select Operator [SEL_10] (rows=11 width=93) - Output:["_col0","_col1"] - Select Operator [SEL_9] (rows=11 width=93) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_18] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1","_col3"] - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - File Output Operator [FS_13] - table:{"name:":"default.dest2"} - Please refer to the previous Select Operator [SEL_9] + Vertex 1 Stage-5 Stats-Aggr Operator Stage-1 @@ -1141,7 +1065,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-4 Stats-Aggr Operator @@ -1151,37 +1075,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 2 vectorized, llap - File Output Operator [FS_27] - table:{"name:":"default.dest2"} - Select Operator [SEL_26] (rows=5 width=93) - Output:["_col0","_col1"] - Group By Operator [GBY_25] (rows=5 width=93) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - File Output Operator [FS_11] - table:{"name:":"default.dest1"} - Merge Join Operator [MERGEJOIN_23] (rows=11 width=93) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=10 width=93) - Output:["_col0"] - Filter Operator [FIL_22] (rows=10 width=93) - predicate:key is not null - TableScan [TS_3] (rows=10 width=93) - default@tbl2,b,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=10 width=93) - Output:["_col0","_col1"] - Filter Operator [FIL_21] (rows=10 width=93) - predicate:key is not null - TableScan [TS_0] (rows=10 width=93) - default@tbl1,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - SHUFFLE [RS_14] - PartitionCols:_col0 - Group By Operator [GBY_13] (rows=11 width=93) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Select Operator [SEL_12] (rows=11 width=93) - Output:["_col0"] - Please refer to the previous Merge Join Operator [MERGEJOIN_23] + Vertex 2 Stage-5 Stats-Aggr Operator Stage-1 diff --git ql/src/test/results/clientpositive/llap/vector_between_columns.q.out ql/src/test/results/clientpositive/llap/vector_between_columns.q.out index 1ed359f..98e8804 100644 --- ql/src/test/results/clientpositive/llap/vector_between_columns.q.out +++ ql/src/test/results/clientpositive/llap/vector_between_columns.q.out @@ -60,7 +60,7 @@ POSTHOOK: Output: default@TINT POSTHOOK: Lineage: tint.cint SIMPLE [(tint_txt)tint_txt.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: tint.rnum SIMPLE [(tint_txt)tint_txt.FieldSchema(name:rnum, type:int, comment:null), ] tint_txt.rnum tint_txt.cint -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col from tint , tsint PREHOOK: type: QUERY @@ -77,10 +77,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tint @@ -97,7 +97,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col1 (type: int), _col3 (type: smallint), CASE WHEN (_col1 BETWEEN _col3 AND _col3) THEN ('Ok') ELSE ('NoOk') END (type: string) @@ -112,7 +112,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tsint @@ -134,7 +134,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col from tint , tsint PREHOOK: type: QUERY PREHOOK: Input: default@tint @@ -171,7 +171,7 @@ tint.rnum tsint.rnum tint.cint tsint.csint between_col 4 2 10 0 NoOk 4 3 10 1 NoOk 4 4 10 10 Ok -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY @@ -188,10 +188,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tint @@ -208,7 +208,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 25 Data size: 385 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col1 BETWEEN _col3 AND _col3 (type: boolean) @@ -226,7 +226,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tsint @@ -248,7 +248,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY PREHOOK: Input: default@tint diff --git ql/src/test/results/clientpositive/llap/vector_between_in.q.out ql/src/test/results/clientpositive/llap/vector_between_in.q.out index 74384a4..215bd6f 100644 --- ql/src/test/results/clientpositive/llap/vector_between_in.q.out +++ ql/src/test/results/clientpositive/llap/vector_between_in.q.out @@ -25,10 +25,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -46,7 +46,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -80,10 +80,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -104,7 +104,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -139,10 +139,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -160,7 +160,7 @@ STAGE PLANS: Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -194,10 +194,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -218,7 +218,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -253,10 +253,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -274,7 +274,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 274112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -308,10 +308,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -329,7 +329,7 @@ STAGE PLANS: Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -363,10 +363,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -384,7 +384,7 @@ STAGE PLANS: Statistics: Num rows: 1365 Data size: 274112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -418,10 +418,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -442,7 +442,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -722,11 +722,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -749,7 +749,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -763,7 +763,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -797,11 +797,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -824,7 +824,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -838,7 +838,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -872,11 +872,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -899,7 +899,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -913,7 +913,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -947,11 +947,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_date_test @@ -974,7 +974,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -988,7 +988,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out index 7de04a7..ca20c11 100644 --- ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out +++ ql/src/test/results/clientpositive/llap/vector_binary_join_groupby.q.out @@ -112,11 +112,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -136,7 +136,7 @@ STAGE PLANS: 1 _col10 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 110 Data size: 32601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18,_col19,_col20,_col21) (type: int) @@ -153,7 +153,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: t2 @@ -173,21 +188,6 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -225,10 +225,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: hundredorc @@ -251,7 +251,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -333,10 +333,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -356,7 +356,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 110 Data size: 32601 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: binary), _col3 (type: binary) @@ -371,7 +371,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 diff --git ql/src/test/results/clientpositive/llap/vector_bucket.q.out ql/src/test/results/clientpositive/llap/vector_bucket.q.out index 0573abf..a68f849 100644 --- ql/src/test/results/clientpositive/llap/vector_bucket.q.out +++ ql/src/test/results/clientpositive/llap/vector_bucket.q.out @@ -23,10 +23,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: values__tmp__table__1 @@ -42,7 +42,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out index 9a2456f..a8ce732 100644 --- ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out +++ ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out @@ -118,11 +118,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -146,7 +146,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -161,7 +161,7 @@ STAGE PLANS: Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: decimal(14,4)) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_char_2.q.out ql/src/test/results/clientpositive/llap/vector_char_2.q.out index bdcf7d8..631ebec 100644 --- ql/src/test/results/clientpositive/llap/vector_char_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_2.q.out @@ -68,11 +68,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -96,7 +96,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -111,7 +111,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -198,11 +198,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: char_2 @@ -226,7 +226,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -241,7 +241,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_char_4.q.out ql/src/test/results/clientpositive/llap/vector_char_4.q.out index 6d55ab0..cedcd8a 100644 --- ql/src/test/results/clientpositive/llap/vector_char_4.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_4.q.out @@ -138,7 +138,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc diff --git ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out index f2a386e..737c4de 100644 --- ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out @@ -137,11 +137,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -161,7 +161,7 @@ STAGE PLANS: 1 _col1 (type: char(10)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -170,7 +170,21 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: char(10)) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: char(10)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -190,20 +204,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: char(10)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -237,11 +237,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -261,7 +261,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: b @@ -281,7 +281,7 @@ STAGE PLANS: 1 _col1 (type: char(20)) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -290,7 +290,7 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: char(20)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -339,11 +339,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 1 <- Vertex 3 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -363,7 +363,7 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) @@ -372,7 +372,21 @@ STAGE PLANS: value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: b @@ -392,20 +406,6 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 323 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out index 18f45ff..de3b155 100644 --- ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out @@ -35,10 +35,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -61,7 +61,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -121,7 +121,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -181,10 +181,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc @@ -207,7 +207,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -267,7 +267,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: str_str_orc diff --git ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out index 5eb4035..5c319b4 100644 --- ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out +++ ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out @@ -46,10 +46,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: m @@ -66,7 +66,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), CASE WHEN ((COALESCE(_col2,5) > 1)) THEN (_col2) ELSE (null) END (type: bigint) @@ -81,7 +81,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: n diff --git ql/src/test/results/clientpositive/llap/vector_complex_all.q.out ql/src/test/results/clientpositive/llap/vector_complex_all.q.out index a7ae10e..fdd627e 100644 --- ql/src/test/results/clientpositive/llap/vector_complex_all.q.out +++ ql/src/test/results/clientpositive/llap/vector_complex_all.q.out @@ -150,7 +150,7 @@ b str two line1 four line2 six line3 -Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_0]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_0]] in Stage 'Vertex 2' is a cross product PREHOOK: query: INSERT INTO TABLE orc_create_complex SELECT orc_create_staging.*, src1.key FROM orc_create_staging cross join src src1 cross join orc_create_staging spam1 cross join orc_create_staging spam2 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vector_complex_join.q.out ql/src/test/results/clientpositive/llap/vector_complex_join.q.out index 89507ee..ab88540 100644 --- ql/src/test/results/clientpositive/llap/vector_complex_join.q.out +++ ql/src/test/results/clientpositive/llap/vector_complex_join.q.out @@ -33,10 +33,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -56,7 +56,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 10090 Data size: 2540021 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -67,7 +67,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test @@ -156,10 +156,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: test2b @@ -175,7 +175,7 @@ STAGE PLANS: 1 a[1] (type: int) outputColumnNames: _col0, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col4 (type: array) @@ -190,7 +190,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: test2a diff --git ql/src/test/results/clientpositive/llap/vector_count.q.out ql/src/test/results/clientpositive/llap/vector_count.q.out index 9ef5c2b..fa54abe 100644 --- ql/src/test/results/clientpositive/llap/vector_count.q.out +++ ql/src/test/results/clientpositive/llap/vector_count.q.out @@ -56,10 +56,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -82,7 +82,7 @@ STAGE PLANS: value expressions: _col5 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -130,10 +130,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -155,7 +155,7 @@ STAGE PLANS: value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -199,10 +199,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -219,7 +219,7 @@ STAGE PLANS: value expressions: d (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -267,10 +267,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: abcd @@ -285,7 +285,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out index 3456d45..e3dfa23 100644 --- ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out +++ ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out @@ -1265,7 +1265,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 3520000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1282,7 +1282,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_data_types.q.out ql/src/test/results/clientpositive/llap/vector_data_types.q.out index 045f536..93a923c 100644 --- ql/src/test/results/clientpositive/llap/vector_data_types.q.out +++ ql/src/test/results/clientpositive/llap/vector_data_types.q.out @@ -108,10 +108,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -128,7 +128,7 @@ STAGE PLANS: value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Select Operator @@ -204,10 +204,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: over1korc @@ -224,7 +224,7 @@ STAGE PLANS: value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_date_1.q.out ql/src/test/results/clientpositive/llap/vector_date_1.q.out index 8fbbb0a..2c11467 100644 --- ql/src/test/results/clientpositive/llap/vector_date_1.q.out +++ ql/src/test/results/clientpositive/llap/vector_date_1.q.out @@ -83,10 +83,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -102,7 +102,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -193,10 +193,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -212,7 +212,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -303,10 +303,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -322,7 +322,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -413,10 +413,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -432,7 +432,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -521,10 +521,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_date_1 @@ -543,7 +543,7 @@ STAGE PLANS: value expressions: _col1 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -654,7 +654,7 @@ STAGE PLANS: value expressions: _col1 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out index b3ff557..e8d5454 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_1.q.out @@ -47,10 +47,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -65,7 +65,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -110,10 +110,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -128,7 +128,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -173,10 +173,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -191,7 +191,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -254,7 +254,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -299,10 +299,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -317,7 +317,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -362,10 +362,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -380,7 +380,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -425,10 +425,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -443,7 +443,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -488,10 +488,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -506,7 +506,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -551,10 +551,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_1 @@ -569,7 +569,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out index 5576078..4540f14 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_10_0.q.out @@ -48,10 +48,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal @@ -66,7 +66,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out index 144356c..baf97e4 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_2.q.out @@ -36,10 +36,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -54,7 +54,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -99,10 +99,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -117,7 +117,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -162,10 +162,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -180,7 +180,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -225,10 +225,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -243,7 +243,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -288,10 +288,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -306,7 +306,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -351,10 +351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -369,7 +369,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -414,10 +414,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -432,7 +432,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -495,7 +495,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -551,10 +551,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -569,7 +569,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -614,10 +614,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -632,7 +632,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -677,10 +677,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -695,7 +695,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -740,10 +740,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -758,7 +758,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -803,10 +803,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -821,7 +821,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -866,10 +866,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -884,7 +884,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -929,10 +929,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -947,7 +947,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -992,10 +992,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_2 @@ -1010,7 +1010,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1070,7 +1070,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1130,7 +1130,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1190,7 +1190,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1250,7 +1250,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1301,7 +1301,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1361,7 +1361,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1421,7 +1421,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1481,7 +1481,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1541,7 +1541,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1601,7 +1601,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1661,7 +1661,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out index a8d1e05..08ffe2e 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_vgby @@ -69,7 +69,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: bigint), _col6 (type: decimal(23,14)), _col7 (type: decimal(23,14)), _col8 (type: decimal(33,14)), _col9 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -148,10 +148,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_vgby @@ -174,7 +174,7 @@ STAGE PLANS: value expressions: _col1 (type: bigint), _col2 (type: decimal(20,10)), _col3 (type: decimal(20,10)), _col4 (type: decimal(30,10)), _col5 (type: struct), _col6 (type: struct), _col7 (type: struct), _col8 (type: bigint), _col9 (type: decimal(23,14)), _col10 (type: decimal(23,14)), _col11 (type: decimal(33,14)), _col12 (type: struct), _col13 (type: struct), _col14 (type: struct), _col15 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out index eddb4dc..d254af0 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_mapjoin.q.out @@ -87,10 +87,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -110,7 +110,7 @@ STAGE PLANS: 1 _col0 (type: decimal(6,2)) outputColumnNames: _col0, _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1153 Data size: 129236 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -121,7 +121,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 diff --git ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out index c16f605..7cf3d3b 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_precision.q.out @@ -581,7 +581,7 @@ STAGE PLANS: value expressions: _col0 (type: struct), _col1 (type: decimal(30,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out index 8e7cd63..d3b65c2 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_round.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_txt @@ -62,7 +62,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -107,10 +107,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_txt @@ -126,7 +126,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -197,10 +197,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_rc @@ -216,7 +216,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -261,10 +261,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_rc @@ -280,7 +280,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: llap LLAP IO: no inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -351,10 +351,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_orc @@ -370,7 +370,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(11,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -415,10 +415,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_orc @@ -434,7 +434,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out index 29ec85b..0d14fca 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_round_2.q.out @@ -47,10 +47,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_1_orc @@ -66,7 +66,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(22,1)), _col3 (type: decimal(23,2)), _col4 (type: decimal(24,3)), _col5 (type: decimal(21,0)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(21,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -160,10 +160,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_2_orc @@ -179,7 +179,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(22,1)), _col3 (type: decimal(23,2)), _col4 (type: decimal(24,3)), _col5 (type: decimal(25,4)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(22,1)), _col13 (type: decimal(23,2)), _col14 (type: decimal(24,3)), _col15 (type: decimal(25,4)), _col16 (type: decimal(21,0)), _col17 (type: decimal(21,0)), _col18 (type: decimal(21,0)), _col19 (type: decimal(21,0)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -300,10 +300,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_3_orc @@ -319,7 +319,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(21,0)), _col2 (type: decimal(21,0)), _col3 (type: decimal(21,0)), _col4 (type: decimal(21,0)), _col5 (type: decimal(21,0)), _col6 (type: decimal(21,0)), _col7 (type: decimal(21,0)), _col8 (type: decimal(21,0)), _col9 (type: decimal(21,0)), _col10 (type: decimal(21,0)), _col11 (type: decimal(21,0)), _col12 (type: decimal(21,0)), _col13 (type: decimal(21,0)), _col14 (type: decimal(21,0)), _col15 (type: decimal(21,0)), _col16 (type: decimal(21,0)), _col17 (type: decimal(22,1)), _col18 (type: decimal(23,2)), _col19 (type: decimal(24,3)), _col20 (type: decimal(25,4)), _col21 (type: decimal(26,5)), _col22 (type: decimal(27,6)), _col23 (type: decimal(28,7)), _col24 (type: decimal(29,8)), _col25 (type: decimal(30,9)), _col26 (type: decimal(31,10)), _col27 (type: decimal(32,11)), _col28 (type: decimal(33,12)), _col29 (type: decimal(34,13)), _col31 (type: decimal(35,14)), _col32 (type: decimal(36,15)), _col33 (type: decimal(37,16)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -429,10 +429,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_tbl_4_orc @@ -448,7 +448,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(30,9)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out index 1be938e..9e13f1a 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out @@ -61,7 +61,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -145,7 +145,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -229,7 +229,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -313,7 +313,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -397,7 +397,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -481,7 +481,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -565,7 +565,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -649,7 +649,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -733,7 +733,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -817,7 +817,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -889,7 +889,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -973,7 +973,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1057,7 +1057,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1141,7 +1141,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1191,7 +1191,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1241,7 +1241,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1324,7 +1324,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1397,7 +1397,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1470,7 +1470,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1554,7 +1554,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1638,11 +1638,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1665,7 +1665,7 @@ STAGE PLANS: value expressions: _col1 (type: decimal(30,10)), _col2 (type: bigint), _col3 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -1683,7 +1683,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(38,18)), _col2 (type: decimal(24,14)), _col3 (type: decimal(30,10)) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1742,7 +1742,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1891,7 +1891,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -1975,7 +1975,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2059,7 +2059,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2143,7 +2143,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2227,7 +2227,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2311,10 +2311,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2337,7 +2337,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2398,10 +2398,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf @@ -2424,7 +2424,7 @@ STAGE PLANS: value expressions: _col1 (type: struct), _col2 (type: struct) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2508,7 +2508,7 @@ STAGE PLANS: value expressions: _col0 (type: array) Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2575,7 +2575,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2642,7 +2642,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2709,7 +2709,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out index 5ea9f4d..3fda0f9 100644 --- ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out +++ ql/src/test/results/clientpositive/llap/vector_decimal_udf2.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf2 @@ -127,7 +127,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: decimal_udf2 diff --git ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out index 620b99e..a9bf4a5 100644 --- ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_distinct_2.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -140,7 +140,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby4.q.out ql/src/test/results/clientpositive/llap/vector_groupby4.q.out index 9ecfa56..58be235 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby4.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby4.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcorc @@ -57,7 +57,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -70,7 +70,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby6.q.out ql/src/test/results/clientpositive/llap/vector_groupby6.q.out index 4c6e038..a860944 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby6.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby6.q.out @@ -37,11 +37,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: srcorc @@ -57,7 +57,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -70,7 +70,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out index 742cebc..9cf9141 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_3.q.out @@ -116,10 +116,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc @@ -142,7 +142,7 @@ STAGE PLANS: value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out index c4bcbab..8c295c4 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -261,7 +261,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -357,11 +357,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -382,7 +382,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -404,7 +404,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -566,11 +566,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -593,7 +593,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -617,7 +617,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -779,11 +779,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -806,7 +806,7 @@ STAGE PLANS: value expressions: _col2 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -830,7 +830,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out index 77a0695..2cb8f4d 100644 --- ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out +++ ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out @@ -142,10 +142,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -166,7 +166,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -227,10 +227,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -251,7 +251,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -315,10 +315,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store @@ -339,7 +339,7 @@ STAGE PLANS: Statistics: Num rows: 24 Data size: 51264 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out index c3c5773..edfbe9f 100644 --- ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out +++ ql/src/test/results/clientpositive/llap/vector_include_no_sel.q.out @@ -162,7 +162,7 @@ POSTHOOK: Lineage: customer_demographics.cd_education_status SIMPLE [(customer_d POSTHOOK: Lineage: customer_demographics.cd_gender SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_gender, type:string, comment:null), ] POSTHOOK: Lineage: customer_demographics.cd_marital_status SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_marital_status, type:string, comment:null), ] POSTHOOK: Lineage: customer_demographics.cd_purchase_estimate SIMPLE [(customer_demographics_txt)customer_demographics_txt.FieldSchema(name:cd_purchase_estimate, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Vertex 2' is a cross product PREHOOK: query: explain select count(1) from customer_demographics,store_sales where ((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'M') or @@ -197,7 +197,7 @@ STAGE PLANS: value expressions: cd_demo_sk (type: int), cd_marital_status (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: store_sales @@ -210,7 +210,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2, _col16 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 200000 Data size: 92055200 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((_col0 = _col16) and (_col2 = 'M')) or ((_col0 = _col16) and (_col2 = 'U'))) (type: boolean) @@ -228,7 +228,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -250,7 +250,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=store_sales] in task 'Vertex 2' is a cross product PREHOOK: query: select count(1) from customer_demographics,store_sales where ((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'M') or (customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk and customer_demographics.cd_marital_status = 'U')) diff --git ql/src/test/results/clientpositive/llap/vector_inner_join.q.out ql/src/test/results/clientpositive/llap/vector_inner_join.q.out index d50123d..dbc9e6d 100644 --- ql/src/test/results/clientpositive/llap/vector_inner_join.q.out +++ ql/src/test/results/clientpositive/llap/vector_inner_join.q.out @@ -43,10 +43,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -66,7 +66,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int) @@ -81,7 +81,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -133,10 +133,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -156,7 +156,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -167,7 +167,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -256,10 +256,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -279,7 +279,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: int) @@ -294,7 +294,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -347,10 +347,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -370,7 +370,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -390,7 +390,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -434,10 +434,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -457,7 +457,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), (_col3 * 2) (type: int), (_col0 * 5) (type: int), _col1 (type: string) @@ -472,7 +472,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -525,10 +525,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -548,7 +548,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col0 (type: int) @@ -563,7 +563,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -616,10 +616,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t2 @@ -639,7 +639,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: int), _col2 (type: string), _col1 (type: string) @@ -654,7 +654,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t1 @@ -707,10 +707,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -730,7 +730,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -750,7 +750,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string), _col2 (type: int) @@ -798,10 +798,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -821,7 +821,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -841,7 +841,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col3 (type: string) diff --git ql/src/test/results/clientpositive/llap/vector_interval_1.q.out ql/src/test/results/clientpositive/llap/vector_interval_1.q.out index 207e9bb..fab8131 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_1.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_1.q.out @@ -61,10 +61,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -80,7 +80,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_year_month), _col4 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -150,10 +150,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -169,7 +169,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_year_month), _col3 (type: interval_year_month), _col5 (type: interval_year_month), _col6 (type: interval_year_month) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -247,10 +247,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -266,7 +266,7 @@ STAGE PLANS: value expressions: _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col5 (type: interval_day_time), _col6 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -356,10 +356,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -375,7 +375,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date), _col7 (type: timestamp), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: timestamp), _col12 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -477,10 +477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -496,7 +496,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp), _col7 (type: timestamp), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: timestamp), _col11 (type: timestamp), _col12 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -580,10 +580,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -599,7 +599,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -665,10 +665,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -684,7 +684,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -756,10 +756,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_1 @@ -775,7 +775,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time), _col5 (type: interval_day_time), _col6 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_interval_2.q.out ql/src/test/results/clientpositive/llap/vector_interval_2.q.out index d84737c..b9f783e 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_2.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_2.q.out @@ -115,10 +115,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -134,7 +134,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean), _col18 (type: boolean), _col19 (type: boolean), _col20 (type: boolean), _col21 (type: boolean), _col22 (type: boolean), _col23 (type: boolean), _col24 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -284,10 +284,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -303,7 +303,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -453,10 +453,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -472,7 +472,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col6 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean), _col18 (type: boolean), _col19 (type: boolean), _col20 (type: boolean), _col21 (type: boolean), _col22 (type: boolean), _col23 (type: boolean), _col24 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -622,10 +622,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -641,7 +641,7 @@ STAGE PLANS: value expressions: _col1 (type: boolean), _col2 (type: boolean), _col3 (type: boolean), _col4 (type: boolean), _col5 (type: boolean), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: boolean), _col10 (type: boolean), _col11 (type: boolean), _col13 (type: boolean), _col14 (type: boolean), _col15 (type: boolean), _col16 (type: boolean), _col17 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -777,10 +777,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -798,7 +798,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -931,10 +931,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -952,7 +952,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1075,10 +1075,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1096,7 +1096,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1219,10 +1219,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1240,7 +1240,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1373,10 +1373,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1394,7 +1394,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1527,10 +1527,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vector_interval_2 @@ -1548,7 +1548,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 394 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out index 391c775..15eb309 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_arithmetic.q.out @@ -69,10 +69,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -88,7 +88,7 @@ STAGE PLANS: value expressions: _col1 (type: date), _col2 (type: date), _col3 (type: date), _col4 (type: date), _col5 (type: date), _col6 (type: date) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -214,10 +214,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -233,7 +233,7 @@ STAGE PLANS: value expressions: _col1 (type: interval_day_time), _col2 (type: interval_day_time), _col3 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -359,10 +359,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -378,7 +378,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -502,10 +502,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -519,7 +519,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -598,10 +598,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -617,7 +617,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -745,10 +745,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -764,7 +764,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: interval_day_time), _col3 (type: interval_day_time), _col4 (type: interval_day_time) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -892,10 +892,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 @@ -911,7 +911,7 @@ STAGE PLANS: value expressions: _col1 (type: timestamp), _col2 (type: timestamp), _col3 (type: timestamp), _col4 (type: timestamp), _col5 (type: timestamp), _col6 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1033,7 +1033,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: interval_arithmetic_1 diff --git ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out index 0bc0e4c..2d5d8a1 100644 --- ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out +++ ql/src/test/results/clientpositive/llap/vector_interval_mapjoin.q.out @@ -189,10 +189,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab_a_1korc @@ -212,7 +212,7 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: interval_day_time) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 1100 Data size: 506290 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time) @@ -227,7 +227,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: vectortab_b_1korc diff --git ql/src/test/results/clientpositive/llap/vector_join30.q.out ql/src/test/results/clientpositive/llap/vector_join30.q.out index d7e5f58..0d38a41 100644 --- ql/src/test/results/clientpositive/llap/vector_join30.q.out +++ ql/src/test/results/clientpositive/llap/vector_join30.q.out @@ -92,7 +92,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -103,7 +103,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -118,7 +118,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -233,7 +252,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 + 1 Vertex 5 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -244,7 +263,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -259,7 +278,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -343,7 +378,19 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Vertex 3 Map Operator Tree: TableScan alias: orcsrc @@ -386,7 +433,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 0 Reducer 2 + 0 Vertex 2 Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -397,7 +444,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -548,8 +595,8 @@ STAGE PLANS: 2 _col0 (type: string) outputColumnNames: _col2, _col3 input vertices: - 1 Reducer 5 - 2 Reducer 7 + 1 Vertex 5 + 2 Vertex 7 Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col2,_col3)) @@ -560,7 +607,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -575,7 +622,26 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Vertex 4 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -588,7 +654,26 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 7 + Vertex 6 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -685,39 +770,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -729,7 +782,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -751,7 +804,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -766,7 +819,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -779,7 +848,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -876,39 +961,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -920,7 +973,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -942,7 +995,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -957,7 +1010,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -970,7 +1039,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1067,39 +1152,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1111,7 +1164,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1133,7 +1186,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1148,7 +1201,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1161,7 +1230,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1258,39 +1343,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: orcsrc - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1302,7 +1355,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1324,7 +1377,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1339,7 +1392,23 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Vertex 5 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 6 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -1352,7 +1421,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Reducer 8 + Vertex 7 + Map Operator Tree: + TableScan + alias: orcsrc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 8 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/llap/vector_join_filters.q.out ql/src/test/results/clientpositive/llap/vector_join_filters.q.out index 1a492b6..2f3e2f6 100644 --- ql/src/test/results/clientpositive/llap/vector_join_filters.q.out +++ ql/src/test/results/clientpositive/llap/vector_join_filters.q.out @@ -26,7 +26,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -36,7 +36,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 3078400 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -46,7 +46,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 4937935 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out index 3497164..689c279 100644 --- ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out +++ ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out @@ -26,7 +26,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -36,7 +36,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -46,7 +46,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Vertex 2' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out index e8dfc30..42aa6f1 100644 --- ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out +++ ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out @@ -46,7 +46,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 104600 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -55,7 +55,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 3418417 Data size: 27347336 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -68,7 +68,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -84,7 +99,7 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -100,21 +115,6 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out index 57d63b3..8475089 100644 --- ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out +++ ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out @@ -91,10 +91,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -114,7 +114,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -129,7 +129,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -181,10 +181,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -204,7 +204,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -219,7 +219,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -271,10 +271,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -294,7 +294,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -309,7 +309,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -361,10 +361,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -384,7 +384,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -399,7 +399,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -451,10 +451,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -474,7 +474,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -489,7 +489,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -541,10 +541,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -564,7 +564,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col4 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: char(2)) @@ -579,7 +579,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 diff --git ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out index 1960c0c..5438873 100644 --- ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out +++ ql/src/test/results/clientpositive/llap/vector_mr_diff_schema_alias.q.out @@ -241,13 +241,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) + Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) + Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: store_sales @@ -267,46 +267,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: unknown - Map 6 - Map Operator Tree: - TableScan - alias: store - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: s_store_sk (type: int), s_state (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 7 - Map Operator Tree: - TableScan - alias: date_dim - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: d_date_sk is not null (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -323,7 +284,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: string) - Reducer 3 + Vertex 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -351,7 +312,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 4 + Vertex 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -366,7 +327,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: bigint) - Reducer 5 + Vertex 5 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator @@ -383,6 +344,45 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 + Map Operator Tree: + TableScan + alias: store + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: ((s_state) IN ('KS', 'AL', 'MN', 'SC', 'VT') and s_store_sk is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: s_store_sk (type: int), s_state (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 7 + Map Operator Tree: + TableScan + alias: date_dim + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: d_date_sk is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: d_date_sk (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_null_projection.q.out ql/src/test/results/clientpositive/llap/vector_null_projection.q.out index a4a36e0..a0e4409 100644 --- ql/src/test/results/clientpositive/llap/vector_null_projection.q.out +++ ql/src/test/results/clientpositive/llap/vector_null_projection.q.out @@ -43,7 +43,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -92,12 +92,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Union 2 (CONTAINS) - Map 4 <- Union 2 (CONTAINS) - Reducer 3 <- Union 2 (SIMPLE_EDGE) + Vertex 1 <- Union 2 (CONTAINS) + Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) + Vertex 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: - Map 1 + Union 2 + Vertex: Union 2 + Vertex 1 Map Operator Tree: TableScan alias: a @@ -118,7 +120,26 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Map 4 + Vertex 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: void) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: null (type: void) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: b @@ -139,27 +160,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: all inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: void) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: null (type: void) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 2 - Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out index c89eb11..b812114 100644 --- ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out +++ ql/src/test/results/clientpositive/llap/vector_number_compare_projection.q.out @@ -114,10 +114,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2k_orc @@ -193,10 +193,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2k_orc diff --git ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out index 503cf5b..d9abd62 100644 --- ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out +++ ql/src/test/results/clientpositive/llap/vector_orderby_5.q.out @@ -116,34 +116,34 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: vectortab2korc - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: bo (type: boolean), b (type: bigint) outputColumnNames: bo, b - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: max(b) keys: bo (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator key expressions: _col0 (type: boolean) sort order: + Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -151,22 +151,22 @@ STAGE PLANS: keys: KEY._col0 (type: boolean) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator key expressions: _col0 (type: boolean) sort order: - - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: boolean), VALUE._col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1000 Data size: 459356 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -186,6 +186,3 @@ POSTHOOK: query: select bo, max(b) from vectortab2korc group by bo order by bo d POSTHOOK: type: QUERY POSTHOOK: Input: default@vectortab2korc #### A masked pattern was here #### -true 9211455920344088576 -false 9209153648361848832 -NULL 9180098147855769600 diff --git ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out index 3f6aca2..6da48e0 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out @@ -73,10 +73,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -93,7 +93,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 6 Data size: 598 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -104,7 +104,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -159,10 +159,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Vertex 2 <- Vertex 1 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: t1 @@ -179,7 +179,7 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: t2 @@ -196,7 +196,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 0 Map 1 + 0 Vertex 1 Statistics: Num rows: 6 Data size: 598 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out index c7cadf1..93b46da 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out @@ -235,10 +235,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -255,7 +255,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 32 Data size: 19648 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -266,7 +266,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: cd @@ -344,10 +344,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -364,7 +364,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 112 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -375,7 +375,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: hd @@ -565,7 +565,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 32 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -575,7 +575,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 240 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col0) @@ -588,7 +588,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -604,7 +619,7 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -620,21 +635,6 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out index a58ce8e..fa040a9 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out @@ -272,7 +272,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 57 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -282,7 +282,7 @@ STAGE PLANS: 1 _col0 (type: bigint) outputColumnNames: _col1 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 162 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col1) @@ -295,7 +295,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -311,7 +326,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -327,21 +342,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), sum(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out index a34cb8d..8854baa 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out @@ -272,7 +272,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -281,7 +281,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 80 Data size: 640 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -294,7 +294,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -310,7 +325,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -326,21 +341,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 979 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -417,7 +417,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 21 Data size: 1869 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -426,7 +426,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 42 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -439,7 +439,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -455,7 +470,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -471,21 +486,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 979 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -562,7 +562,7 @@ STAGE PLANS: 1 _col0 (type: bigint), _col1 (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 20 Data size: 1860 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -571,7 +571,7 @@ STAGE PLANS: 0 _col0 (type: int), _col2 (type: string) 1 _col0 (type: int), _col1 (type: string) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -584,7 +584,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -600,7 +615,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 2120 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -616,21 +631,6 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1023 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out index 0fc7021..42bf428 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out @@ -265,10 +265,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -285,7 +285,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 81 Data size: 49734 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -296,7 +296,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: cd @@ -409,10 +409,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: c @@ -429,7 +429,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 450 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -440,7 +440,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: hd @@ -935,7 +935,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 81 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -944,7 +944,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 1215 Data size: 9720 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -957,7 +957,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -973,7 +988,7 @@ STAGE PLANS: Statistics: Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 + Vertex 4 Map Operator Tree: TableScan alias: hd @@ -989,21 +1004,6 @@ STAGE PLANS: Statistics: Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out index 6866862..8bc7a19 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out @@ -105,7 +105,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -118,7 +118,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: st @@ -134,21 +149,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -221,7 +221,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -234,7 +234,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -250,21 +265,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -337,7 +337,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -350,7 +350,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -366,21 +381,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -453,7 +453,7 @@ STAGE PLANS: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 6444 Data size: 51552 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -466,7 +466,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -482,21 +497,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -572,7 +572,7 @@ STAGE PLANS: 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: tinyint) @@ -581,39 +581,7 @@ STAGE PLANS: Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: sm - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col0 pmod 8) (type: bigint) - sort order: + - Map-reduce partition columns: (_col0 pmod 8) (type: bigint) - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -632,7 +600,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -647,6 +615,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col0 pmod 8) (type: bigint) + sort order: + + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator @@ -782,7 +782,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -795,7 +795,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: st @@ -811,21 +826,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -898,7 +898,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -911,7 +911,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -927,21 +942,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1014,7 +1014,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1027,7 +1027,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -1043,21 +1058,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1130,7 +1130,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 40386 Data size: 323088 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1143,7 +1143,22 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: sm @@ -1159,21 +1174,6 @@ STAGE PLANS: Statistics: Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1249,7 +1249,7 @@ STAGE PLANS: 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Vertex 4 Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) @@ -1258,39 +1258,7 @@ STAGE PLANS: Statistics: Num rows: 151450 Data size: 605800 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: sm - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col0 pmod 8) (type: bigint) - sort order: + - Map-reduce partition columns: (_col0 pmod 8) (type: bigint) - Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Map 5 - Map Operator Tree: - TableScan - alias: s2 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cmodtinyint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: vectorized, llap - LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -1309,7 +1277,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 + Vertex 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -1324,6 +1292,38 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: sm + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col0 pmod 8) (type: bigint) + sort order: + + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) + Statistics: Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Vertex 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cmodtinyint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out index 736b8f9..420cf28 100644 --- ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out +++ ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out @@ -139,10 +139,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -159,7 +159,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) @@ -173,7 +173,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) @@ -188,7 +188,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -205,7 +205,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: tjoin3 @@ -264,10 +264,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE), Vertex 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: tjoin1 @@ -284,7 +284,7 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col2, _col3 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) @@ -298,7 +298,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 4 Data size: 449 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -309,7 +309,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: tjoin2 @@ -326,7 +326,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs - Map 3 + Vertex 3 Map Operator Tree: TableScan alias: tjoin3 diff --git ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out index 8b054b8..991cd5f 100644 --- ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out +++ ql/src/test/results/clientpositive/llap/vector_partition_diff_num_cols.q.out @@ -110,7 +110,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -234,7 +234,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -358,7 +358,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -469,7 +469,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -580,7 +580,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs - Reducer 2 + Vertex 2 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/perf/query1.q.out ql/src/test/results/clientpositive/perf/query1.q.out index a47548f..17906aa 100644 --- ql/src/test/results/clientpositive/perf/query1.q.out +++ ql/src/test/results/clientpositive/perf/query1.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[119][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 18' is a cross product +Warning: Shuffle Join MERGEJOIN[119][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 18' is a cross product PREHOOK: query: explain with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk @@ -48,202 +48,24 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 18 <- Map 17 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Reducer 23 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (PARTITION_EDGE), Vertex 20 (PARTITION_EDGE), Vertex 23 (PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_84] - Limit [LIM_83] (rows=100 width=2) - Number of rows:100 - Select Operator [SEL_82] (rows=435393727038609152 width=2) - Output:["_col0"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_81] - Select Operator [SEL_80] (rows=435393727038609152 width=2) - Output:["_col0"] - Filter Operator [FIL_79] (rows=435393727038609152 width=2) - predicate:(_col2 > CASE WHEN (_col8 is null) THEN (null) ELSE (_col7) END) - Merge Join Operator [MERGEJOIN_125] (rows=1306181181115827456 width=2) - Conds:RS_75._col1=RS_76._col2(Left Outer),Output:["_col2","_col6","_col7","_col8"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col2 - Select Operator [SEL_68] (rows=1187437411641012224 width=2) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_67] (rows=1187437411641012224 width=2) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_65] (rows=2374874823282024448 width=2) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col3 - Select Operator [SEL_64] (rows=2374874823282024448 width=2) - Output:["_col3","_col2"] - Merge Join Operator [MERGEJOIN_124] (rows=2374874823282024448 width=2) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col2","_col3"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Select Operator [SEL_33] (rows=31675133 width=77) - Output:["_col1","_col2"] - Group By Operator [GBY_32] (rows=31675133 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0, _col1 - Group By Operator [GBY_30] (rows=63350266 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_29] (rows=63350266 width=77) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_123] (rows=63350266 width=77) - Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_114] (rows=57591150 width=77) - predicate:sr_returned_date_sk is not null - TableScan [TS_20] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_customer_sk","sr_store_sk","sr_fee"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_115] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Group By Operator [GBY_59] (rows=2158977065280000000 width=2) - Output:["_col0"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=4317954130560000000 width=2) - Output:["_col0"],keys:_col3 - Merge Join Operator [MERGEJOIN_119] (rows=4317954130560000000 width=2) - Conds:(Inner),(Inner),Output:["_col3"] - <-Map 17 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Select Operator [SEL_35] (rows=80000000 width=4) - TableScan [TS_34] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:COMPLETE - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_53] - Select Operator [SEL_37] (rows=1704 width=4) - TableScan [TS_36] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 23 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_54] - Select Operator [SEL_51] (rows=31675133 width=77) - Output:["_col1"] - Group By Operator [GBY_50] (rows=31675133 width=77) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0, _col1 - Group By Operator [GBY_48] (rows=63350266 width=77) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_118] (rows=63350266 width=77) - Conds:RS_44._col0=RS_45._col0(Inner),Output:["_col1","_col2"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_40] (rows=57591150 width=77) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_116] (rows=57591150 width=77) - predicate:sr_returned_date_sk is not null - TableScan [TS_38] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_customer_sk","sr_store_sk"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_43] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_117] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_41] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_122] (rows=88000001 width=860) - Conds:RS_72._col0=RS_73._col0(Inner),Output:["_col1","_col2","_col6"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_113] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_17] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_121] (rows=34842647 width=77) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col0","_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_112] (rows=852 width=1910) - predicate:((s_state = 'NM') and s_store_sk is not null) - TableScan [TS_14] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Select Operator [SEL_13] (rows=31675133 width=77) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_12] (rows=31675133 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=63350266 width=77) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=63350266 width=77) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_120] (rows=63350266 width=77) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_110] (rows=57591150 width=77) - predicate:(sr_returned_date_sk is not null and sr_store_sk is not null and sr_customer_sk is not null) - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_customer_sk","sr_store_sk","sr_fee"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_111] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query12.q.out ql/src/test/results/clientpositive/perf/query12.q.out index afe2cd4..dfa9e6e 100644 --- ql/src/test/results/clientpositive/perf/query12.q.out +++ ql/src/test/results/clientpositive/perf/query12.q.out @@ -23,70 +23,15 @@ and date_dim.d_date between cast('2001-01-12' as date) group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio limit 100 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_26] - Limit [LIM_25] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_24] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_23] - Select Operator [SEL_21] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_20] (rows=21780404 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_19] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Group By Operator [GBY_17] (rows=21780404 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_15] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col23)"],keys:_col38, _col41, _col49, _col47, _col42 - Select Operator [SEL_14] (rows=43560808 width=135) - Output:["_col38","_col41","_col49","_col47","_col42","_col23"] - Filter Operator [FIL_32] (rows=43560808 width=135) - predicate:((_col3 = _col37) and (_col0 = _col62)) - Merge Join Operator [MERGEJOIN_37] (rows=174243235 width=135) - Conds:RS_9._col0=RS_11.d_date_sk(Inner),Output:["_col0","_col3","_col23","_col37","_col38","_col41","_col42","_col47","_col49","_col62"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:d_date_sk - Filter Operator [FIL_35] (rows=8116 width=1119) - predicate:(d_date_sk is not null and d_date BETWEEN 2001-01-12 AND 2001-02-11 00:00:00.0) - TableScan [TS_2] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_36] (rows=158402938 width=135) - Conds:RS_4.ws_item_sk=RS_6.i_item_sk(Inner),Output:["_col0","_col3","_col23","_col37","_col38","_col41","_col42","_col47","_col49"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:ws_item_sk - Filter Operator [FIL_33] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:i_item_sk - Filter Operator [FIL_34] (rows=231000 width=1436) - predicate:(i_item_sk is not null and (i_category) IN ('Jewelry', 'Sports', 'Books')) - TableScan [TS_1] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query13.q.out ql/src/test/results/clientpositive/perf/query13.q.out index 941a3a1..ea15591 100644 --- ql/src/test/results/clientpositive/perf/query13.q.out +++ ql/src/test/results/clientpositive/perf/query13.q.out @@ -109,109 +109,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 12 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_43] - Group By Operator [GBY_41] (rows=1 width=764) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","sum(VALUE._col3)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_40] - Group By Operator [GBY_39] (rows=1 width=764) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col7)","avg(_col9)","avg(_col10)","sum(_col10)"] - Merge Join Operator [MERGEJOIN_74] (rows=2016666 width=1014) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col7","_col9","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_64] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=1833333 width=1014) - Output:["_col0","_col5","_col7","_col8"] - Filter Operator [FIL_33] (rows=1833333 width=1014) - predicate:(((_col17) IN ('KY', 'GA', 'NM') and _col9 BETWEEN 100 AND 200) or ((_col17) IN ('MT', 'OR', 'IN') and _col9 BETWEEN 150 AND 300) or ((_col17) IN ('WI', 'MO', 'WV') and _col9 BETWEEN 50 AND 250)) - Merge Join Operator [MERGEJOIN_73] (rows=11000000 width=1014) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col0","_col5","_col7","_col8","_col9","_col17"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_29] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_69] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_27] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_26] (rows=3549306 width=88) - Output:["_col0","_col3","_col5","_col7","_col8","_col9"] - Filter Operator [FIL_25] (rows=3549306 width=88) - predicate:(((_col12 = 'M') and (_col13 = '4 yr Degree') and _col6 BETWEEN 100 AND 150 and (_col15 = 3)) or ((_col12 = 'D') and (_col13 = 'Primary') and _col6 BETWEEN 50 AND 100 and (_col15 = 1)) or ((_col12 = 'U') and (_col13 = 'Advanced Degree') and _col6 BETWEEN 150 AND 200 and (_col15 = 1))) - Select Operator [SEL_24] (rows=85183356 width=88) - Output:["_col0","_col3","_col5","_col6","_col7","_col8","_col9","_col12","_col13","_col15"] - Merge Join Operator [MERGEJOIN_72] (rows=85183356 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col4","_col6","_col7","_col8","_col9","_col10","_col12","_col14","_col15"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_68] (rows=1861800 width=385) - predicate:(((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and ((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_71] (rows=77439413 width=88) - Conds:RS_18._col3=RS_19._col0(Inner),Output:["_col1","_col2","_col4","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_67] (rows=7200 width=107) - predicate:(((hd_dep_count = 3) or (hd_dep_count = 1)) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_70] (rows=70399465 width=88) - Conds:RS_15._col0=RS_16._col4(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9","_col10"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1704 width=1910) - Output:["_col0"] - Filter Operator [FIL_65] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_3] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col4 - Select Operator [SEL_8] (rows=63999513 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_66] (rows=63999513 width=88) - predicate:((ss_sales_price BETWEEN 100 AND 150 or ss_sales_price BETWEEN 50 AND 100 or ss_sales_price BETWEEN 150 AND 200) and (ss_net_profit BETWEEN 100 AND 200 or ss_net_profit BETWEEN 150 AND 300 or ss_net_profit BETWEEN 50 AND 250) and ss_store_sk is not null and ss_cdemo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_quantity","ss_sales_price","ss_ext_sales_price","ss_ext_wholesale_cost","ss_net_profit"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query14.q.out ql/src/test/results/clientpositive/perf/query14.q.out index 21cabbc..41ff9fc 100644 --- ql/src/test/results/clientpositive/perf/query14.q.out +++ ql/src/test/results/clientpositive/perf/query14.q.out @@ -1,9 +1,9 @@ -Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 114' is a cross product -Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 115' is a cross product -Warning: Shuffle Join MERGEJOIN[912][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[913][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product -Warning: Shuffle Join MERGEJOIN[914][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 61' is a cross product -Warning: Shuffle Join MERGEJOIN[915][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 62' is a cross product +Warning: Shuffle Join MERGEJOIN[914][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 61' is a cross product +Warning: Shuffle Join MERGEJOIN[915][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 62' is a cross product +Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 114' is a cross product +Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 115' is a cross product +Warning: Shuffle Join MERGEJOIN[912][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 5' is a cross product +Warning: Shuffle Join MERGEJOIN[913][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk @@ -213,1356 +213,87 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 100 <- Map 103 (SIMPLE_EDGE), Reducer 99 (SIMPLE_EDGE) -Reducer 101 <- Reducer 100 (SIMPLE_EDGE), Union 94 (CONTAINS) -Reducer 105 <- Map 104 (SIMPLE_EDGE), Map 108 (SIMPLE_EDGE) -Reducer 106 <- Map 109 (SIMPLE_EDGE), Reducer 105 (SIMPLE_EDGE) -Reducer 107 <- Reducer 106 (SIMPLE_EDGE), Union 94 (CONTAINS) -Reducer 111 <- Map 110 (SIMPLE_EDGE), Map 116 (SIMPLE_EDGE), Union 112 (CONTAINS) -Reducer 113 <- Union 112 (CUSTOM_SIMPLE_EDGE) -Reducer 114 <- Reducer 113 (CUSTOM_SIMPLE_EDGE), Reducer 126 (CUSTOM_SIMPLE_EDGE) -Reducer 115 <- Reducer 114 (CUSTOM_SIMPLE_EDGE), Reducer 137 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 118 <- Map 117 (SIMPLE_EDGE), Map 119 (SIMPLE_EDGE), Union 112 (CONTAINS) -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 121 <- Map 120 (SIMPLE_EDGE), Map 122 (SIMPLE_EDGE), Union 112 (CONTAINS) -Reducer 124 <- Map 123 (SIMPLE_EDGE), Map 127 (SIMPLE_EDGE), Union 125 (CONTAINS) -Reducer 126 <- Union 125 (CUSTOM_SIMPLE_EDGE) -Reducer 129 <- Map 128 (SIMPLE_EDGE), Map 130 (SIMPLE_EDGE), Union 125 (CONTAINS) -Reducer 132 <- Map 131 (SIMPLE_EDGE), Map 133 (SIMPLE_EDGE), Union 125 (CONTAINS) -Reducer 135 <- Map 134 (SIMPLE_EDGE), Map 138 (SIMPLE_EDGE) -Reducer 136 <- Map 139 (SIMPLE_EDGE), Reducer 135 (SIMPLE_EDGE), Reducer 142 (SIMPLE_EDGE) -Reducer 137 <- Reducer 136 (SIMPLE_EDGE) -Reducer 141 <- Map 140 (SIMPLE_EDGE), Reducer 148 (SIMPLE_EDGE) -Reducer 142 <- Reducer 141 (SIMPLE_EDGE) -Reducer 144 <- Map 143 (SIMPLE_EDGE), Map 149 (SIMPLE_EDGE) -Reducer 145 <- Map 150 (SIMPLE_EDGE), Reducer 144 (SIMPLE_EDGE) -Reducer 146 <- Reducer 145 (SIMPLE_EDGE), Union 147 (CONTAINS) -Reducer 148 <- Union 147 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 152 <- Map 151 (SIMPLE_EDGE), Map 155 (SIMPLE_EDGE) -Reducer 153 <- Map 156 (SIMPLE_EDGE), Reducer 152 (SIMPLE_EDGE) -Reducer 154 <- Reducer 153 (SIMPLE_EDGE), Union 147 (CONTAINS) -Reducer 158 <- Map 157 (SIMPLE_EDGE), Map 161 (SIMPLE_EDGE) -Reducer 159 <- Map 162 (SIMPLE_EDGE), Reducer 158 (SIMPLE_EDGE) -Reducer 160 <- Reducer 159 (SIMPLE_EDGE), Union 147 (CONTAINS) -Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 19 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 20 <- Union 19 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE), Union 19 (CONTAINS) -Reducer 26 <- Map 25 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE), Union 19 (CONTAINS) -Reducer 29 <- Map 28 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE) -Reducer 30 <- Map 33 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 31 <- Reducer 30 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Reducer 42 (SIMPLE_EDGE) -Reducer 36 <- Reducer 35 (SIMPLE_EDGE) -Reducer 38 <- Map 37 (SIMPLE_EDGE), Map 43 (SIMPLE_EDGE) -Reducer 39 <- Map 44 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) -Reducer 40 <- Reducer 39 (SIMPLE_EDGE), Union 41 (CONTAINS) -Reducer 42 <- Union 41 (SIMPLE_EDGE) -Reducer 46 <- Map 45 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE) -Reducer 47 <- Map 50 (SIMPLE_EDGE), Reducer 46 (SIMPLE_EDGE) -Reducer 48 <- Reducer 47 (SIMPLE_EDGE), Union 41 (CONTAINS) -Reducer 5 <- Reducer 20 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 52 <- Map 51 (SIMPLE_EDGE), Map 55 (SIMPLE_EDGE) -Reducer 53 <- Map 56 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 54 <- Reducer 53 (SIMPLE_EDGE), Union 41 (CONTAINS) -Reducer 58 <- Map 57 (SIMPLE_EDGE), Map 63 (SIMPLE_EDGE), Union 59 (CONTAINS) -Reducer 6 <- Reducer 31 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 60 <- Union 59 (CUSTOM_SIMPLE_EDGE) -Reducer 61 <- Reducer 60 (CUSTOM_SIMPLE_EDGE), Reducer 73 (CUSTOM_SIMPLE_EDGE) -Reducer 62 <- Reducer 61 (CUSTOM_SIMPLE_EDGE), Reducer 84 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 65 <- Map 64 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE), Union 59 (CONTAINS) -Reducer 68 <- Map 67 (SIMPLE_EDGE), Map 69 (SIMPLE_EDGE), Union 59 (CONTAINS) -Reducer 71 <- Map 70 (SIMPLE_EDGE), Map 74 (SIMPLE_EDGE), Union 72 (CONTAINS) -Reducer 73 <- Union 72 (CUSTOM_SIMPLE_EDGE) -Reducer 76 <- Map 75 (SIMPLE_EDGE), Map 77 (SIMPLE_EDGE), Union 72 (CONTAINS) -Reducer 79 <- Map 78 (SIMPLE_EDGE), Map 80 (SIMPLE_EDGE), Union 72 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) -Reducer 82 <- Map 81 (SIMPLE_EDGE), Map 85 (SIMPLE_EDGE) -Reducer 83 <- Map 86 (SIMPLE_EDGE), Reducer 82 (SIMPLE_EDGE), Reducer 89 (SIMPLE_EDGE) -Reducer 84 <- Reducer 83 (SIMPLE_EDGE) -Reducer 88 <- Map 87 (SIMPLE_EDGE), Reducer 95 (SIMPLE_EDGE) -Reducer 89 <- Reducer 88 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) -Reducer 91 <- Map 90 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE) -Reducer 92 <- Map 97 (SIMPLE_EDGE), Reducer 91 (SIMPLE_EDGE) -Reducer 93 <- Reducer 92 (SIMPLE_EDGE), Union 94 (CONTAINS) -Reducer 95 <- Union 94 (SIMPLE_EDGE) -Reducer 99 <- Map 102 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) +Vertex 100 <- Vertex 103 (SORT_PARTITION_EDGE), Vertex 99 (SORT_PARTITION_EDGE) +Vertex 101 <- Union 94 (CONTAINS), Vertex 100 (SORT_PARTITION_EDGE) +Vertex 105 <- Vertex 104 (SORT_PARTITION_EDGE), Vertex 108 (SORT_PARTITION_EDGE) +Vertex 106 <- Vertex 105 (SORT_PARTITION_EDGE), Vertex 109 (SORT_PARTITION_EDGE) +Vertex 107 <- Union 94 (CONTAINS), Vertex 106 (SORT_PARTITION_EDGE) +Vertex 111 <- Union 112 (CONTAINS), Vertex 110 (SORT_PARTITION_EDGE), Vertex 116 (SORT_PARTITION_EDGE) +Vertex 113 <- Union 112 (PARTITION_EDGE) +Vertex 114 <- Vertex 113 (PARTITION_EDGE), Vertex 126 (PARTITION_EDGE) +Vertex 115 <- Union 7 (CONTAINS), Vertex 114 (PARTITION_EDGE), Vertex 137 (PARTITION_EDGE) +Vertex 118 <- Union 112 (CONTAINS), Vertex 117 (SORT_PARTITION_EDGE), Vertex 119 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 121 <- Union 112 (CONTAINS), Vertex 120 (SORT_PARTITION_EDGE), Vertex 122 (SORT_PARTITION_EDGE) +Vertex 124 <- Union 125 (CONTAINS), Vertex 123 (SORT_PARTITION_EDGE), Vertex 127 (SORT_PARTITION_EDGE) +Vertex 126 <- Union 125 (PARTITION_EDGE) +Vertex 129 <- Union 125 (CONTAINS), Vertex 128 (SORT_PARTITION_EDGE), Vertex 130 (SORT_PARTITION_EDGE) +Vertex 132 <- Union 125 (CONTAINS), Vertex 131 (SORT_PARTITION_EDGE), Vertex 133 (SORT_PARTITION_EDGE) +Vertex 135 <- Vertex 134 (SORT_PARTITION_EDGE), Vertex 138 (SORT_PARTITION_EDGE) +Vertex 136 <- Vertex 135 (SORT_PARTITION_EDGE), Vertex 139 (SORT_PARTITION_EDGE), Vertex 142 (SORT_PARTITION_EDGE) +Vertex 137 <- Vertex 136 (SORT_PARTITION_EDGE) +Vertex 141 <- Vertex 140 (SORT_PARTITION_EDGE), Vertex 148 (SORT_PARTITION_EDGE) +Vertex 142 <- Vertex 141 (SORT_PARTITION_EDGE) +Vertex 144 <- Vertex 143 (SORT_PARTITION_EDGE), Vertex 149 (SORT_PARTITION_EDGE) +Vertex 145 <- Vertex 144 (SORT_PARTITION_EDGE), Vertex 150 (SORT_PARTITION_EDGE) +Vertex 146 <- Union 147 (CONTAINS), Vertex 145 (SORT_PARTITION_EDGE) +Vertex 148 <- Union 147 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 3 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 152 <- Vertex 151 (SORT_PARTITION_EDGE), Vertex 155 (SORT_PARTITION_EDGE) +Vertex 153 <- Vertex 152 (SORT_PARTITION_EDGE), Vertex 156 (SORT_PARTITION_EDGE) +Vertex 154 <- Union 147 (CONTAINS), Vertex 153 (SORT_PARTITION_EDGE) +Vertex 158 <- Vertex 157 (SORT_PARTITION_EDGE), Vertex 161 (SORT_PARTITION_EDGE) +Vertex 159 <- Vertex 158 (SORT_PARTITION_EDGE), Vertex 162 (SORT_PARTITION_EDGE) +Vertex 160 <- Union 147 (CONTAINS), Vertex 159 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 19 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 19 (PARTITION_EDGE) +Vertex 23 <- Union 19 (CONTAINS), Vertex 22 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 26 <- Union 19 (CONTAINS), Vertex 25 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 42 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE), Vertex 43 (SORT_PARTITION_EDGE) +Vertex 39 <- Vertex 38 (SORT_PARTITION_EDGE), Vertex 44 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (PARTITION_EDGE) +Vertex 40 <- Union 41 (CONTAINS), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 42 <- Union 41 (SORT_PARTITION_EDGE) +Vertex 46 <- Vertex 45 (SORT_PARTITION_EDGE), Vertex 49 (SORT_PARTITION_EDGE) +Vertex 47 <- Vertex 46 (SORT_PARTITION_EDGE), Vertex 50 (SORT_PARTITION_EDGE) +Vertex 48 <- Union 41 (CONTAINS), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 20 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE), Vertex 56 (SORT_PARTITION_EDGE) +Vertex 54 <- Union 41 (CONTAINS), Vertex 53 (SORT_PARTITION_EDGE) +Vertex 58 <- Union 59 (CONTAINS), Vertex 57 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 7 (CONTAINS), Vertex 31 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 60 <- Union 59 (PARTITION_EDGE) +Vertex 61 <- Vertex 60 (PARTITION_EDGE), Vertex 73 (PARTITION_EDGE) +Vertex 62 <- Union 7 (CONTAINS), Vertex 61 (PARTITION_EDGE), Vertex 84 (PARTITION_EDGE) +Vertex 65 <- Union 59 (CONTAINS), Vertex 64 (SORT_PARTITION_EDGE), Vertex 66 (SORT_PARTITION_EDGE) +Vertex 68 <- Union 59 (CONTAINS), Vertex 67 (SORT_PARTITION_EDGE), Vertex 69 (SORT_PARTITION_EDGE) +Vertex 71 <- Union 72 (CONTAINS), Vertex 70 (SORT_PARTITION_EDGE), Vertex 74 (SORT_PARTITION_EDGE) +Vertex 73 <- Union 72 (PARTITION_EDGE) +Vertex 76 <- Union 72 (CONTAINS), Vertex 75 (SORT_PARTITION_EDGE), Vertex 77 (SORT_PARTITION_EDGE) +Vertex 79 <- Union 72 (CONTAINS), Vertex 78 (SORT_PARTITION_EDGE), Vertex 80 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 82 <- Vertex 81 (SORT_PARTITION_EDGE), Vertex 85 (SORT_PARTITION_EDGE) +Vertex 83 <- Vertex 82 (SORT_PARTITION_EDGE), Vertex 86 (SORT_PARTITION_EDGE), Vertex 89 (SORT_PARTITION_EDGE) +Vertex 84 <- Vertex 83 (SORT_PARTITION_EDGE) +Vertex 88 <- Vertex 87 (SORT_PARTITION_EDGE), Vertex 95 (SORT_PARTITION_EDGE) +Vertex 89 <- Vertex 88 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) +Vertex 91 <- Vertex 90 (SORT_PARTITION_EDGE), Vertex 96 (SORT_PARTITION_EDGE) +Vertex 92 <- Vertex 91 (SORT_PARTITION_EDGE), Vertex 97 (SORT_PARTITION_EDGE) +Vertex 93 <- Union 94 (CONTAINS), Vertex 92 (SORT_PARTITION_EDGE) +Vertex 95 <- Union 94 (SORT_PARTITION_EDGE) +Vertex 99 <- Vertex 102 (SORT_PARTITION_EDGE), Vertex 98 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 9 - File Output Operator [FS_596] - Limit [LIM_595] (rows=100 width=406) - Number of rows:100 - Select Operator [SEL_594] (rows=1016388080 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_593] - Select Operator [SEL_592] (rows=1016388080 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_591] (rows=1016388080 width=406) - Output:["_col0","_col1","_col2","_col3","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 115 [CONTAINS] - Reduce Output Operator [RS_590] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_589] (rows=2032776160 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_586] (rows=58081078 width=433) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_585] (rows=58081078 width=433) - predicate:(_col5 > _col1) - Merge Join Operator [MERGEJOIN_917] (rows=174243235 width=433) - Conds:(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 114 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_582] - Merge Join Operator [MERGEJOIN_916] (rows=1 width=297) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 113 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_579] - Select Operator [SEL_436] (rows=1 width=8) - Filter Operator [FIL_435] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_433] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_428] (rows=1 width=8) - Group By Operator [GBY_427] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Union 112 [CUSTOM_SIMPLE_EDGE] - <-Reducer 111 [CONTAINS] - Reduce Output Operator [RS_426] - Group By Operator [GBY_425] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_424] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_401] (rows=633595212 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_897] (rows=633595212 width=88) - Conds:RS_398._col0=RS_399._col0(Inner),Output:["_col1"] - <-Map 110 [SIMPLE_EDGE] - SHUFFLE [RS_398] - PartitionCols:_col0 - Select Operator [SEL_394] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_842] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_392] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"] - <-Map 116 [SIMPLE_EDGE] - SHUFFLE [RS_399] - PartitionCols:_col0 - Select Operator [SEL_397] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_843] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_395] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 118 [CONTAINS] - Reduce Output Operator [RS_426] - Group By Operator [GBY_425] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_424] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_411] (rows=316788826 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_898] (rows=316788826 width=135) - Conds:RS_408._col0=RS_409._col0(Inner),Output:["_col1"] - <-Map 117 [SIMPLE_EDGE] - SHUFFLE [RS_408] - PartitionCols:_col0 - Select Operator [SEL_404] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_844] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_402] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"] - <-Map 119 [SIMPLE_EDGE] - SHUFFLE [RS_409] - PartitionCols:_col0 - Select Operator [SEL_407] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_845] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_405] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 121 [CONTAINS] - Reduce Output Operator [RS_426] - Group By Operator [GBY_425] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_424] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_423] (rows=158402938 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_899] (rows=158402938 width=135) - Conds:RS_420._col0=RS_421._col0(Inner),Output:["_col1"] - <-Map 120 [SIMPLE_EDGE] - SHUFFLE [RS_420] - PartitionCols:_col0 - Select Operator [SEL_416] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_846] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_414] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"] - <-Map 122 [SIMPLE_EDGE] - SHUFFLE [RS_421] - PartitionCols:_col0 - Select Operator [SEL_419] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_847] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_417] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 126 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_580] - Group By Operator [GBY_473] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Union 125 [CUSTOM_SIMPLE_EDGE] - <-Reducer 124 [CONTAINS] - Reduce Output Operator [RS_472] - Group By Operator [GBY_471] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_469] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_446] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_900] (rows=633595212 width=88) - Conds:RS_443._col0=RS_444._col0(Inner),Output:["_col1","_col2"] - <-Map 123 [SIMPLE_EDGE] - SHUFFLE [RS_443] - PartitionCols:_col0 - Select Operator [SEL_439] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_848] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_437] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"] - <-Map 127 [SIMPLE_EDGE] - SHUFFLE [RS_444] - PartitionCols:_col0 - Select Operator [SEL_442] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_849] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_440] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 129 [CONTAINS] - Reduce Output Operator [RS_472] - Group By Operator [GBY_471] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_469] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_456] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_901] (rows=316788826 width=135) - Conds:RS_453._col0=RS_454._col0(Inner),Output:["_col1","_col2"] - <-Map 128 [SIMPLE_EDGE] - SHUFFLE [RS_453] - PartitionCols:_col0 - Select Operator [SEL_449] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_850] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_447] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"] - <-Map 130 [SIMPLE_EDGE] - SHUFFLE [RS_454] - PartitionCols:_col0 - Select Operator [SEL_452] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_851] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_450] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 132 [CONTAINS] - Reduce Output Operator [RS_472] - Group By Operator [GBY_471] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_469] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_468] (rows=158402938 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_902] (rows=158402938 width=135) - Conds:RS_465._col0=RS_466._col0(Inner),Output:["_col1","_col2"] - <-Map 131 [SIMPLE_EDGE] - SHUFFLE [RS_465] - PartitionCols:_col0 - Select Operator [SEL_461] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_852] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_459] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"] - <-Map 133 [SIMPLE_EDGE] - SHUFFLE [RS_466] - PartitionCols:_col0 - Select Operator [SEL_464] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_853] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_462] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 137 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_583] - Group By Operator [GBY_577] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 136 [SIMPLE_EDGE] - SHUFFLE [RS_576] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_575] (rows=348486471 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_573] (rows=348486471 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_911] (rows=348486471 width=135) - Conds:RS_569._col1=RS_570._col0(Inner),RS_569._col1=RS_571._col0(Inner),Output:["_col2","_col3","_col8","_col9","_col10"] - <-Map 139 [SIMPLE_EDGE] - SHUFFLE [RS_570] - PartitionCols:_col0 - Select Operator [SEL_483] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_856] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_481] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 135 [SIMPLE_EDGE] - SHUFFLE [RS_569] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_903] (rows=158402938 width=135) - Conds:RS_566._col0=RS_567._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 134 [SIMPLE_EDGE] - SHUFFLE [RS_566] - PartitionCols:_col0 - Select Operator [SEL_477] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_854] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_475] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_quantity","ws_list_price"] - <-Map 138 [SIMPLE_EDGE] - SHUFFLE [RS_567] - PartitionCols:_col0 - Select Operator [SEL_480] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_855] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_478] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 142 [SIMPLE_EDGE] - SHUFFLE [RS_571] - PartitionCols:_col0 - Group By Operator [GBY_564] (rows=254100 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Reducer 141 [SIMPLE_EDGE] - SHUFFLE [RS_563] - PartitionCols:_col0 - Group By Operator [GBY_562] (rows=508200 width=1436) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_910] (rows=508200 width=1436) - Conds:RS_558._col1, _col2, _col3=RS_559._col0, _col1, _col2(Inner),Output:["_col0"] - <-Map 140 [SIMPLE_EDGE] - SHUFFLE [RS_558] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_486] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_857] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) - TableScan [TS_484] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 148 [SIMPLE_EDGE] - SHUFFLE [RS_559] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_557] (rows=1 width=108) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_556] (rows=1 width=108) - predicate:(_col3 = 3) - Group By Operator [GBY_555] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 147 [SIMPLE_EDGE] - <-Reducer 146 [CONTAINS] - Reduce Output Operator [RS_554] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_553] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_506] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 145 [SIMPLE_EDGE] - SHUFFLE [RS_505] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_504] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_502] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_905] (rows=696954748 width=88) - Conds:RS_499._col1=RS_500._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 150 [SIMPLE_EDGE] - SHUFFLE [RS_500] - PartitionCols:_col0 - Select Operator [SEL_495] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_860] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_493] (rows=462000 width=1436) - default@item,iss,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 144 [SIMPLE_EDGE] - SHUFFLE [RS_499] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_904] (rows=633595212 width=88) - Conds:RS_496._col0=RS_497._col0(Inner),Output:["_col1"] - <-Map 143 [SIMPLE_EDGE] - SHUFFLE [RS_496] - PartitionCols:_col0 - Select Operator [SEL_489] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_858] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_487] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 149 [SIMPLE_EDGE] - SHUFFLE [RS_497] - PartitionCols:_col0 - Select Operator [SEL_492] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_859] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_490] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 154 [CONTAINS] - Reduce Output Operator [RS_554] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_553] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_527] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 153 [SIMPLE_EDGE] - SHUFFLE [RS_526] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_525] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_523] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_907] (rows=348467716 width=135) - Conds:RS_520._col1=RS_521._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 156 [SIMPLE_EDGE] - SHUFFLE [RS_521] - PartitionCols:_col0 - Select Operator [SEL_516] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_863] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_514] (rows=462000 width=1436) - default@item,ics,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 152 [SIMPLE_EDGE] - SHUFFLE [RS_520] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_906] (rows=316788826 width=135) - Conds:RS_517._col0=RS_518._col0(Inner),Output:["_col1"] - <-Map 151 [SIMPLE_EDGE] - SHUFFLE [RS_517] - PartitionCols:_col0 - Select Operator [SEL_510] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_861] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_508] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] - <-Map 155 [SIMPLE_EDGE] - SHUFFLE [RS_518] - PartitionCols:_col0 - Select Operator [SEL_513] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_862] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_511] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 160 [CONTAINS] - Reduce Output Operator [RS_554] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_553] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_549] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 159 [SIMPLE_EDGE] - SHUFFLE [RS_548] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_547] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_545] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_909] (rows=174243235 width=135) - Conds:RS_542._col1=RS_543._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 162 [SIMPLE_EDGE] - SHUFFLE [RS_543] - PartitionCols:_col0 - Select Operator [SEL_538] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_866] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_536] (rows=462000 width=1436) - default@item,iws,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 158 [SIMPLE_EDGE] - SHUFFLE [RS_542] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_908] (rows=158402938 width=135) - Conds:RS_539._col0=RS_540._col0(Inner),Output:["_col1"] - <-Map 157 [SIMPLE_EDGE] - SHUFFLE [RS_539] - PartitionCols:_col0 - Select Operator [SEL_532] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_864] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_530] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"] - <-Map 161 [SIMPLE_EDGE] - SHUFFLE [RS_540] - PartitionCols:_col0 - Select Operator [SEL_535] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_865] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_533] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 6 [CONTAINS] - Reduce Output Operator [RS_590] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_589] (rows=2032776160 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_194] (rows=232318249 width=386) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_193] (rows=232318249 width=386) - predicate:(_col5 > _col1) - Merge Join Operator [MERGEJOIN_913] (rows=696954748 width=386) - Conds:(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 31 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_191] - Group By Operator [GBY_185] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_184] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_183] (rows=1393909496 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_181] (rows=1393909496 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_881] (rows=1393909496 width=88) - Conds:RS_177._col1=RS_178._col0(Inner),RS_177._col1=RS_179._col0(Inner),Output:["_col2","_col3","_col8","_col9","_col10"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_806] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_89] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_177] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_873] (rows=633595212 width=88) - Conds:RS_174._col0=RS_175._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_174] - PartitionCols:_col0 - Select Operator [SEL_85] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_804] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_83] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_quantity","ss_list_price"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_805] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_86] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_179] - PartitionCols:_col0 - Group By Operator [GBY_172] (rows=254100 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_171] - PartitionCols:_col0 - Group By Operator [GBY_170] (rows=508200 width=1436) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_880] (rows=508200 width=1436) - Conds:RS_166._col1, _col2, _col3=RS_167._col0, _col1, _col2(Inner),Output:["_col0"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_166] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_94] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_807] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) - TableScan [TS_92] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 42 [SIMPLE_EDGE] - SHUFFLE [RS_167] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_165] (rows=1 width=108) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_164] (rows=1 width=108) - predicate:(_col3 = 3) - Group By Operator [GBY_163] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 41 [SIMPLE_EDGE] - <-Reducer 40 [CONTAINS] - Reduce Output Operator [RS_162] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_161] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_114] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 39 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_112] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_110] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_875] (rows=696954748 width=88) - Conds:RS_107._col1=RS_108._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 44 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_103] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_810] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_101] (rows=462000 width=1436) - default@item,iss,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_874] (rows=633595212 width=88) - Conds:RS_104._col0=RS_105._col0(Inner),Output:["_col1"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Select Operator [SEL_97] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_808] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_95] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 43 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_809] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_98] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 48 [CONTAINS] - Reduce Output Operator [RS_162] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_161] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_135] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 47 [SIMPLE_EDGE] - SHUFFLE [RS_134] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_133] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_131] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_877] (rows=348467716 width=135) - Conds:RS_128._col1=RS_129._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 50 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col0 - Select Operator [SEL_124] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_813] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_122] (rows=462000 width=1436) - default@item,ics,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 46 [SIMPLE_EDGE] - SHUFFLE [RS_128] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_876] (rows=316788826 width=135) - Conds:RS_125._col0=RS_126._col0(Inner),Output:["_col1"] - <-Map 45 [SIMPLE_EDGE] - SHUFFLE [RS_125] - PartitionCols:_col0 - Select Operator [SEL_118] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_811] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_116] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] - <-Map 49 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0 - Select Operator [SEL_121] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_812] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_119] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 54 [CONTAINS] - Reduce Output Operator [RS_162] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_161] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_157] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 53 [SIMPLE_EDGE] - SHUFFLE [RS_156] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_155] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_153] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_879] (rows=174243235 width=135) - Conds:RS_150._col1=RS_151._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_151] - PartitionCols:_col0 - Select Operator [SEL_146] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_816] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_144] (rows=462000 width=1436) - default@item,iws,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_150] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_878] (rows=158402938 width=135) - Conds:RS_147._col0=RS_148._col0(Inner),Output:["_col1"] - <-Map 51 [SIMPLE_EDGE] - SHUFFLE [RS_147] - PartitionCols:_col0 - Select Operator [SEL_140] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_814] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_138] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col0 - Select Operator [SEL_143] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_815] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_141] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_190] - Merge Join Operator [MERGEJOIN_912] (rows=1 width=297) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_188] - Group By Operator [GBY_81] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Union 19 [CUSTOM_SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_80] - Group By Operator [GBY_79] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_77] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_54] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_870] (rows=633595212 width=88) - Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_798] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_45] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_799] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_48] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 23 [CONTAINS] - Reduce Output Operator [RS_80] - Group By Operator [GBY_79] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_77] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_64] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_871] (rows=316788826 width=135) - Conds:RS_61._col0=RS_62._col0(Inner),Output:["_col1","_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_800] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_55] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_60] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_801] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_58] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 26 [CONTAINS] - Reduce Output Operator [RS_80] - Group By Operator [GBY_79] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_77] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_76] (rows=158402938 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_872] (rows=158402938 width=135) - Conds:RS_73._col0=RS_74._col0(Inner),Output:["_col1","_col2"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_802] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_67] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_803] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_70] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_187] - Select Operator [SEL_44] (rows=1 width=8) - Filter Operator [FIL_43] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_41] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_36] (rows=1 width=8) - Group By Operator [GBY_35] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Union 3 [CUSTOM_SIMPLE_EDGE] - <-Reducer 12 [CONTAINS] - Reduce Output Operator [RS_34] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_32] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_19] (rows=316788826 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_868] (rows=316788826 width=135) - Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col1"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_794] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_10] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_795] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_34] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_32] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_31] (rows=158402938 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_869] (rows=158402938 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_796] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_22] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_797] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [CONTAINS] - Reduce Output Operator [RS_34] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_32] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_867] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_792] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_793] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 62 [CONTAINS] - Reduce Output Operator [RS_590] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_589] (rows=2032776160 width=406) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_389] (rows=116155905 width=433) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_388] (rows=116155905 width=433) - predicate:(_col5 > _col1) - Merge Join Operator [MERGEJOIN_915] (rows=348467716 width=433) - Conds:(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 61 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_385] - Merge Join Operator [MERGEJOIN_914] (rows=1 width=297) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 60 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_382] - Select Operator [SEL_239] (rows=1 width=8) - Filter Operator [FIL_238] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_236] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_231] (rows=1 width=8) - Group By Operator [GBY_230] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Union 59 [CUSTOM_SIMPLE_EDGE] - <-Reducer 58 [CONTAINS] - Reduce Output Operator [RS_229] - Group By Operator [GBY_228] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_227] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_204] (rows=633595212 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_882] (rows=633595212 width=88) - Conds:RS_201._col0=RS_202._col0(Inner),Output:["_col1"] - <-Map 57 [SIMPLE_EDGE] - SHUFFLE [RS_201] - PartitionCols:_col0 - Select Operator [SEL_197] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_817] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_195] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_202] - PartitionCols:_col0 - Select Operator [SEL_200] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_818] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_198] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 65 [CONTAINS] - Reduce Output Operator [RS_229] - Group By Operator [GBY_228] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_227] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_214] (rows=316788826 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_883] (rows=316788826 width=135) - Conds:RS_211._col0=RS_212._col0(Inner),Output:["_col1"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_211] - PartitionCols:_col0 - Select Operator [SEL_207] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_819] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_205] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"] - <-Map 66 [SIMPLE_EDGE] - SHUFFLE [RS_212] - PartitionCols:_col0 - Select Operator [SEL_210] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_820] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_208] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 68 [CONTAINS] - Reduce Output Operator [RS_229] - Group By Operator [GBY_228] (rows=1 width=8) - Output:["_col0"],aggregations:["count(_col0)"] - Select Operator [SEL_227] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_226] (rows=158402938 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_884] (rows=158402938 width=135) - Conds:RS_223._col0=RS_224._col0(Inner),Output:["_col1"] - <-Map 67 [SIMPLE_EDGE] - SHUFFLE [RS_223] - PartitionCols:_col0 - Select Operator [SEL_219] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_821] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_217] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"] - <-Map 69 [SIMPLE_EDGE] - SHUFFLE [RS_224] - PartitionCols:_col0 - Select Operator [SEL_222] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_822] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_220] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 73 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_383] - Group By Operator [GBY_276] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Union 72 [CUSTOM_SIMPLE_EDGE] - <-Reducer 71 [CONTAINS] - Reduce Output Operator [RS_275] - Group By Operator [GBY_274] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_272] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_249] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_885] (rows=633595212 width=88) - Conds:RS_246._col0=RS_247._col0(Inner),Output:["_col1","_col2"] - <-Map 70 [SIMPLE_EDGE] - SHUFFLE [RS_246] - PartitionCols:_col0 - Select Operator [SEL_242] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_823] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_240] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"] - <-Map 74 [SIMPLE_EDGE] - SHUFFLE [RS_247] - PartitionCols:_col0 - Select Operator [SEL_245] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_824] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_243] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 76 [CONTAINS] - Reduce Output Operator [RS_275] - Group By Operator [GBY_274] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_272] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_259] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_886] (rows=316788826 width=135) - Conds:RS_256._col0=RS_257._col0(Inner),Output:["_col1","_col2"] - <-Map 75 [SIMPLE_EDGE] - SHUFFLE [RS_256] - PartitionCols:_col0 - Select Operator [SEL_252] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_825] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_250] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"] - <-Map 77 [SIMPLE_EDGE] - SHUFFLE [RS_257] - PartitionCols:_col0 - Select Operator [SEL_255] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_826] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_253] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 79 [CONTAINS] - Reduce Output Operator [RS_275] - Group By Operator [GBY_274] (rows=1 width=288) - Output:["_col0"],aggregations:["avg(_col0)"] - Select Operator [SEL_272] (rows=1108786976 width=108) - Output:["_col0"] - Select Operator [SEL_271] (rows=158402938 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_887] (rows=158402938 width=135) - Conds:RS_268._col0=RS_269._col0(Inner),Output:["_col1","_col2"] - <-Map 78 [SIMPLE_EDGE] - SHUFFLE [RS_268] - PartitionCols:_col0 - Select Operator [SEL_264] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_827] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_262] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"] - <-Map 80 [SIMPLE_EDGE] - SHUFFLE [RS_269] - PartitionCols:_col0 - Select Operator [SEL_267] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_828] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - TableScan [TS_265] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 84 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_386] - Group By Operator [GBY_380] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 83 [SIMPLE_EDGE] - SHUFFLE [RS_379] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_378] (rows=696935432 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_376] (rows=696935432 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_896] (rows=696935432 width=135) - Conds:RS_372._col1=RS_373._col0(Inner),RS_372._col1=RS_374._col0(Inner),Output:["_col2","_col3","_col8","_col9","_col10"] - <-Map 86 [SIMPLE_EDGE] - SHUFFLE [RS_373] - PartitionCols:_col0 - Select Operator [SEL_286] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_831] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_284] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 82 [SIMPLE_EDGE] - SHUFFLE [RS_372] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_888] (rows=316788826 width=135) - Conds:RS_369._col0=RS_370._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 81 [SIMPLE_EDGE] - SHUFFLE [RS_369] - PartitionCols:_col0 - Select Operator [SEL_280] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_829] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_278] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_quantity","cs_list_price"] - <-Map 85 [SIMPLE_EDGE] - SHUFFLE [RS_370] - PartitionCols:_col0 - Select Operator [SEL_283] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_830] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_281] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 89 [SIMPLE_EDGE] - SHUFFLE [RS_374] - PartitionCols:_col0 - Group By Operator [GBY_367] (rows=254100 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Reducer 88 [SIMPLE_EDGE] - SHUFFLE [RS_366] - PartitionCols:_col0 - Group By Operator [GBY_365] (rows=508200 width=1436) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_895] (rows=508200 width=1436) - Conds:RS_361._col1, _col2, _col3=RS_362._col0, _col1, _col2(Inner),Output:["_col0"] - <-Map 87 [SIMPLE_EDGE] - SHUFFLE [RS_361] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_289] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_832] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) - TableScan [TS_287] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 95 [SIMPLE_EDGE] - SHUFFLE [RS_362] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_360] (rows=1 width=108) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_359] (rows=1 width=108) - predicate:(_col3 = 3) - Group By Operator [GBY_358] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 94 [SIMPLE_EDGE] - <-Reducer 101 [CONTAINS] - Reduce Output Operator [RS_357] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_356] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_330] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 100 [SIMPLE_EDGE] - SHUFFLE [RS_329] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_328] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_326] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_892] (rows=348467716 width=135) - Conds:RS_323._col1=RS_324._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 103 [SIMPLE_EDGE] - SHUFFLE [RS_324] - PartitionCols:_col0 - Select Operator [SEL_319] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_838] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_317] (rows=462000 width=1436) - default@item,ics,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 99 [SIMPLE_EDGE] - SHUFFLE [RS_323] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_891] (rows=316788826 width=135) - Conds:RS_320._col0=RS_321._col0(Inner),Output:["_col1"] - <-Map 102 [SIMPLE_EDGE] - SHUFFLE [RS_321] - PartitionCols:_col0 - Select Operator [SEL_316] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_837] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_314] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 98 [SIMPLE_EDGE] - SHUFFLE [RS_320] - PartitionCols:_col0 - Select Operator [SEL_313] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_836] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_311] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] - <-Reducer 107 [CONTAINS] - Reduce Output Operator [RS_357] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_356] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_352] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 106 [SIMPLE_EDGE] - SHUFFLE [RS_351] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_350] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_348] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_894] (rows=174243235 width=135) - Conds:RS_345._col1=RS_346._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 109 [SIMPLE_EDGE] - SHUFFLE [RS_346] - PartitionCols:_col0 - Select Operator [SEL_341] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_841] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_339] (rows=462000 width=1436) - default@item,iws,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 105 [SIMPLE_EDGE] - SHUFFLE [RS_345] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_893] (rows=158402938 width=135) - Conds:RS_342._col0=RS_343._col0(Inner),Output:["_col1"] - <-Map 104 [SIMPLE_EDGE] - SHUFFLE [RS_342] - PartitionCols:_col0 - Select Operator [SEL_335] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_839] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_333] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"] - <-Map 108 [SIMPLE_EDGE] - SHUFFLE [RS_343] - PartitionCols:_col0 - Select Operator [SEL_338] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_840] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_336] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 93 [CONTAINS] - Reduce Output Operator [RS_357] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_356] (rows=609832849 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_309] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 92 [SIMPLE_EDGE] - SHUFFLE [RS_308] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_307] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Select Operator [SEL_305] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_890] (rows=696954748 width=88) - Conds:RS_302._col1=RS_303._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 97 [SIMPLE_EDGE] - SHUFFLE [RS_303] - PartitionCols:_col0 - Select Operator [SEL_298] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_835] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) - TableScan [TS_296] (rows=462000 width=1436) - default@item,iss,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 91 [SIMPLE_EDGE] - SHUFFLE [RS_302] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_889] (rows=633595212 width=88) - Conds:RS_299._col0=RS_300._col0(Inner),Output:["_col1"] - <-Map 90 [SIMPLE_EDGE] - SHUFFLE [RS_299] - PartitionCols:_col0 - Select Operator [SEL_292] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_833] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_290] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 96 [SIMPLE_EDGE] - SHUFFLE [RS_300] - PartitionCols:_col0 - Select Operator [SEL_295] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_834] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - TableScan [TS_293] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 9 diff --git ql/src/test/results/clientpositive/perf/query15.q.out ql/src/test/results/clientpositive/perf/query15.q.out index 4f4dcc5..53e9ce7 100644 --- ql/src/test/results/clientpositive/perf/query15.q.out +++ ql/src/test/results/clientpositive/perf/query15.q.out @@ -5,83 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_31] - Limit [LIM_30] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_29] (rows=174233858 width=135) - Output:["_col0","_col1"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_28] - Group By Operator [GBY_26] (rows=174233858 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=348467716 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_23] (rows=348467716 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_47] (rows=348467716 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col2","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_44] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=316788826 width=135) - Output:["_col0","_col2","_col7"] - Filter Operator [FIL_15] (rows=316788826 width=135) - predicate:((substr(_col4, 1, 5)) IN ('85669', '86197', '88274', '83405', '86475', '85392', '85460', '80348', '81792') or (_col3) IN ('CA', 'WA', 'GA') or (_col7 > 500)) - Merge Join Operator [MERGEJOIN_46] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col1(Inner),Output:["_col3","_col4","_col5","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_43] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_6] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_45] (rows=88000001 width=860) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_zip"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query16.q.out ql/src/test/results/clientpositive/perf/query16.q.out index 1be5fba..0bf40e8 100644 --- ql/src/test/results/clientpositive/perf/query16.q.out +++ ql/src/test/results/clientpositive/perf/query16.q.out @@ -1,5 +1,5 @@ -Warning: Shuffle Join MERGEJOIN[141][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 13' is a cross product -Warning: Shuffle Join MERGEJOIN[142][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 25' is a cross product +Warning: Shuffle Join MERGEJOIN[141][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Vertex 13' is a cross product +Warning: Shuffle Join MERGEJOIN[142][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Vertex 25' is a cross product PREHOOK: query: explain select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` @@ -61,216 +61,25 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (CUSTOM_SIMPLE_EDGE), Map 17 (CUSTOM_SIMPLE_EDGE), Map 18 (CUSTOM_SIMPLE_EDGE), Map 19 (CUSTOM_SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (CUSTOM_SIMPLE_EDGE), Map 27 (CUSTOM_SIMPLE_EDGE), Map 28 (CUSTOM_SIMPLE_EDGE), Map 29 (CUSTOM_SIMPLE_EDGE) -Reducer 26 <- Reducer 25 (SIMPLE_EDGE) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 16 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 23 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (PARTITION_EDGE), Vertex 17 (PARTITION_EDGE), Vertex 18 (PARTITION_EDGE), Vertex 19 (PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE), Vertex 28 (PARTITION_EDGE), Vertex 29 (PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_94] - Limit [LIM_93] (rows=1 width=344) - Number of rows:100 - Select Operator [SEL_92] (rows=1 width=344) - Output:["_col0","_col1","_col2"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_91] - Group By Operator [GBY_89] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_88] - Group By Operator [GBY_87] (rows=1534538622412395776 width=1) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col4)","sum(_col5)","sum(_col6)"],keys:_col4 - Select Operator [SEL_86] (rows=1534538622412395776 width=1) - Output:["_col4","_col5","_col6"] - Filter Operator [FIL_85] (rows=1534538622412395776 width=1) - predicate:_col16 is null - Select Operator [SEL_84] (rows=3069077244824791552 width=1) - Output:["_col4","_col5","_col6","_col16"] - Merge Join Operator [MERGEJOIN_149] (rows=3069077244824791552 width=1) - Conds:RS_81._col3, _col4=RS_82._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0, _col1 - Group By Operator [GBY_67] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0, _col1 - Group By Operator [GBY_65] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],keys:_col2, _col3 - Select Operator [SEL_64] (rows=5072854730221289472 width=1) - Output:["_col2","_col3"] - Filter Operator [FIL_63] (rows=5072854730221289472 width=1) - predicate:(_col2 <> _col0) - Merge Join Operator [MERGEJOIN_147] (rows=5072854730221289472 width=1) - Conds:RS_60._col1=RS_61._col1(Inner),Output:["_col0","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=287989836 width=135) - Output:["_col0","_col1"] - TableScan [TS_40] (rows=287989836 width=135) - default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Select Operator [SEL_59] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_58] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0, _col1 - Group By Operator [GBY_56] (rows=9223372036854775807 width=1) - Output:["_col0","_col1"],keys:_col4, _col3 - Select Operator [SEL_55] (rows=9223372036854775807 width=1) - Output:["_col4","_col3"] - Merge Join Operator [MERGEJOIN_142] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"] - <-Map 24 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_50] - Select Operator [SEL_43] (rows=73049 width=4) - TableScan [TS_42] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE - <-Map 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_51] - Select Operator [SEL_45] (rows=60 width=4) - TableScan [TS_44] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE - <-Map 28 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Select Operator [SEL_47] (rows=40000000 width=4) - TableScan [TS_46] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Map 29 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_53] - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1"] - TableScan [TS_48] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col3, _col4 - Merge Join Operator [MERGEJOIN_148] (rows=2790070162094850048 width=1) - Conds:RS_78._col4=RS_79._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_38] (rows=2536427365110644736 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Group By Operator [GBY_36] (rows=5072854730221289472 width=1) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_146] (rows=5072854730221289472 width=1) - Conds:RS_32._col0=RS_33._col0(Inner),Output:["_col0"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=28798881 width=106) - Output:["_col0"] - TableScan [TS_30] (rows=28798881 width=106) - default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_26] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col3 - Merge Join Operator [MERGEJOIN_141] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),Output:["_col3"] - <-Map 12 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_20] - Select Operator [SEL_13] (rows=73049 width=4) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE - <-Map 17 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_21] - Select Operator [SEL_15] (rows=60 width=4) - TableScan [TS_14] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE - <-Map 18 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_22] - Select Operator [SEL_17] (rows=40000000 width=4) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Map 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Select Operator [SEL_19] (rows=287989836 width=135) - Output:["_col0"] - TableScan [TS_18] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_order_number"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_78] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_145] (rows=383314495 width=135) - Conds:RS_75._col2=RS_76._col0(Inner),Output:["_col3","_col4","_col5","_col6"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=30 width=2045) - Output:["_col0"] - Filter Operator [FIL_136] (rows=30 width=2045) - predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null) - TableScan [TS_9] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_144] (rows=348467716 width=135) - Conds:RS_72._col1=RS_73._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_135] (rows=20000000 width=1014) - predicate:((ca_state = 'NY') and ca_address_sk is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_143] (rows=316788826 width=135) - Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_133] (rows=287989836 width=135) - predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_134] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-4-01' AND 2001-05-31 01:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query17.q.out ql/src/test/results/clientpositive/perf/query17.q.out index f03aeaa..3d3e265 100644 --- ql/src/test/results/clientpositive/perf/query17.q.out +++ ql/src/test/results/clientpositive/perf/query17.q.out @@ -5,143 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Select Operator [SEL_50] (rows=510205767 width=88) - Output:["_col0","_col1","_col10","_col11","_col12","_col13","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["count(VALUE._col0)","avg(VALUE._col1)","stddev_samp(VALUE._col2)","count(VALUE._col3)","avg(VALUE._col4)","stddev_samp(VALUE._col5)","count(VALUE._col6)","avg(VALUE._col7)","stddev_samp(VALUE._col8)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["count(_col15)","avg(_col15)","stddev_samp(_col15)","count(_col20)","avg(_col20)","stddev_samp(_col20)","count(_col3)","avg(_col3)","stddev_samp(_col3)"],keys:_col24, _col25, _col7 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col24","_col25","_col7","_col15","_col20","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col11=RS_44._col0(Inner),Output:["_col3","_col7","_col15","_col20","_col24","_col25"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col11 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col12, _col11(Inner),Output:["_col3","_col7","_col11","_col15","_col20"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col12, _col11 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col14","_col5","_col9"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col16"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=36525 width=1119) - predicate:((d_quarter_name) IN ('2000Q1', '2000Q2', '2000Q3') and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=36524 width=1119) - predicate:((d_quarter_name = '2000Q1') and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_return_quantity"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_quantity"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=36525 width=1119) - predicate:((d_quarter_name) IN ('2000Q1', '2000Q2', '2000Q3') and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_quarter_name"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query18.q.out ql/src/test/results/clientpositive/perf/query18.q.out index cf11954..339f4a7 100644 --- ql/src/test/results/clientpositive/perf/query18.q.out +++ ql/src/test/results/clientpositive/perf/query18.q.out @@ -5,128 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_49] - Limit [LIM_48] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_47] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_46] - Select Operator [SEL_45] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Group By Operator [GBY_44] (rows=1054114882 width=135) - Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)","avg(VALUE._col4)","avg(VALUE._col5)","avg(VALUE._col6)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_42] (rows=2108229765 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],aggregations:["avg(_col4)","avg(_col5)","avg(_col6)","avg(_col7)","avg(_col8)","avg(_col9)","avg(_col10)"],keys:_col0, _col1, _col2, _col3, 0 - Select Operator [SEL_40] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Merge Join Operator [MERGEJOIN_83] (rows=421645953 width=135) - Conds:RS_37._col0=RS_38._col3(Inner),Output:["_col4","_col6","_col7","_col8","_col11","_col16","_col17","_col18","_col19","_col20","_col26"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col3 - Select Operator [SEL_30] (rows=383314495 width=135) - Output:["_col1","_col10","_col16","_col3","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_82] (rows=383314495 width=135) - Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col8","_col14","_col16"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_18] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_81] (rows=348467716 width=135) - Conds:RS_24._col2=RS_25._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col14"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=465450 width=385) - Output:["_col0","_col3"] - Filter Operator [FIL_76] (rows=465450 width=385) - predicate:((cd_gender = 'M') and (cd_education_status = 'College') and cd_demo_sk is not null) - TableScan [TS_15] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_education_status","cd_dep_count"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_80] (rows=316788826 width=135) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_75] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Filter Operator [FIL_74] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_cdemo_sk is not null and cs_item_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_9] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_bill_cdemo_sk","cs_item_sk","cs_quantity","cs_list_price","cs_sales_price","cs_coupon_amt","cs_net_profit"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_79] (rows=48400001 width=860) - Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col0","_col4","_col6","_col7","_col8"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_73] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_6] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_78] (rows=44000000 width=860) - Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col0","_col1","_col4","_col6","_col7","_col8"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=40000000 width=860) - Output:["_col0","_col1","_col2","_col4"] - Filter Operator [FIL_71] (rows=40000000 width=860) - predicate:((c_birth_month) IN (9, 5, 12, 4, 1, 10) and c_customer_sk is not null and c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk","c_birth_month","c_birth_year"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_72] (rows=20000000 width=1014) - predicate:((ca_state) IN ('ND', 'WI', 'AL', 'NC', 'OK', 'MS', 'TN') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_state","ca_country"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query19.q.out ql/src/test/results/clientpositive/perf/query19.q.out index 0ddcd83..9f445cc 100644 --- ql/src/test/results/clientpositive/perf/query19.q.out +++ ql/src/test/results/clientpositive/perf/query19.q.out @@ -5,117 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_45] - Limit [LIM_44] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_43] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - Select Operator [SEL_40] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_39] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_37] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col7)"],keys:_col10, _col9, _col11, _col12 - Select Operator [SEL_36] (rows=843315281 width=88) - Output:["_col10","_col9","_col11","_col12","_col7"] - Filter Operator [FIL_35] (rows=843315281 width=88) - predicate:(substr(_col17, 1, 5) <> substr(_col19, 1, 5)) - Select Operator [SEL_34] (rows=843315281 width=88) - Output:["_col7","_col9","_col10","_col11","_col12","_col17","_col19"] - Merge Join Operator [MERGEJOIN_73] (rows=843315281 width=88) - Conds:RS_31._col0=RS_32._col2(Inner),Output:["_col3","_col8","_col13","_col14","_col15","_col16","_col19"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col2 - Select Operator [SEL_27] (rows=766650239 width=88) - Output:["_col10","_col11","_col12","_col15","_col2","_col4","_col9"] - Merge Join Operator [MERGEJOIN_72] (rows=766650239 width=88) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col2","_col4","_col9","_col10","_col11","_col12","_col15"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_68] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_zip"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_71] (rows=696954748 width=88) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col3","_col4","_col9","_col10","_col11","_col12"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_67] (rows=231000 width=1436) - predicate:((i_manager_id = 7) and i_item_sk is not null) - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manufact_id","i_manufact","i_manager_id"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_70] (rows=633595212 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_66] (rows=18262 width=1119) - predicate:((d_moy = 11) and (d_year = 1999) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_65] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_customer_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ext_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_69] (rows=88000001 width=860) - Conds:RS_28._col1=RS_29._col0(Inner),Output:["_col0","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_63] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_64] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_zip"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query20.q.out ql/src/test/results/clientpositive/perf/query20.q.out index c525703..3ba37dd 100644 --- ql/src/test/results/clientpositive/perf/query20.q.out +++ ql/src/test/results/clientpositive/perf/query20.q.out @@ -5,76 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_26] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Select Operator [SEL_23] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_22] (rows=174233858 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_21] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col3 - Select Operator [SEL_19] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_18] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_16] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)"],keys:_col10, _col9, _col6, _col7, _col8 - Select Operator [SEL_15] (rows=348467716 width=135) - Output:["_col10","_col9","_col6","_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_38] (rows=348467716 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_36] (rows=231000 width=1436) - predicate:((i_category) IN ('Jewelry', 'Sports', 'Books') and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_37] (rows=316788826 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_34] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_35] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2001-01-12' AND '2001-02-11' and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query21.q.out ql/src/test/results/clientpositive/perf/query21.q.out index 67a49e2..75de1c4 100644 --- ql/src/test/results/clientpositive/perf/query21.q.out +++ ql/src/test/results/clientpositive/perf/query21.q.out @@ -59,81 +59,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_31] - Limit [LIM_30] (rows=100 width=15) - Number of rows:100 - Select Operator [SEL_29] (rows=12506076 width=15) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_28] - Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col2 > 0)) THEN ((UDFToDouble(_col3) / UDFToDouble(_col2)) BETWEEN 0.6666666666666666 AND 1.5) ELSE (null) END - Group By Operator [GBY_25] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1 - Group By Operator [GBY_23] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_47] (rows=50024305 width=15) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col7","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_44] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_9] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_46] (rows=45476640 width=15) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=51333 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_43] (rows=51333 width=1436) - predicate:(i_current_price BETWEEN 0.99 AND 1.49 and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_current_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_45] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_41] (rows=37584000 width=15) - predicate:(inv_warehouse_sk is not null and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-03-08 23:00:00.0 AND 1998-05-08 00:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query22.q.out ql/src/test/results/clientpositive/perf/query22.q.out index 6ff8896..03ceb09 100644 --- ql/src/test/results/clientpositive/perf/query22.q.out +++ ql/src/test/results/clientpositive/perf/query22.q.out @@ -43,81 +43,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_30] - Limit [LIM_29] (rows=100 width=15) - Number of rows:100 - Select Operator [SEL_28] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - Select Operator [SEL_25] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=125060762 width=15) - Output:["_col0","_col1","_col2","_col3","_col5"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_22] (rows=250121525 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(_col3)"],keys:_col8, _col9, _col10, _col11, 0 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col8","_col9","_col10","_col11","_col3"] - Merge Join Operator [MERGEJOIN_46] (rows=50024305 width=15) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col8","_col9","_col10","_col11"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_43] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category","i_product_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_45] (rows=45476640 width=15) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=27 width=1029) - Output:["_col0"] - Filter Operator [FIL_42] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_6] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_44] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=37584000 width=15) - predicate:(inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_41] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query23.q.out ql/src/test/results/clientpositive/perf/query23.q.out index 59f2586..7b350c4 100644 --- ql/src/test/results/clientpositive/perf/query23.q.out +++ ql/src/test/results/clientpositive/perf/query23.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join MERGEJOIN[382][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 51' is a cross product -Warning: Shuffle Join MERGEJOIN[383][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 52' is a cross product -Warning: Shuffle Join MERGEJOIN[379][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 19' is a cross product -Warning: Shuffle Join MERGEJOIN[380][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 20' is a cross product +Warning: Shuffle Join MERGEJOIN[379][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 19' is a cross product +Warning: Shuffle Join MERGEJOIN[380][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 20' is a cross product +Warning: Shuffle Join MERGEJOIN[382][tables = [$hdt$_1, $hdt$_2]] in Stage 'Vertex 51' is a cross product +Warning: Shuffle Join MERGEJOIN[383][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Vertex 52' is a cross product PREHOOK: query: explain with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales @@ -105,553 +105,49 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 16 <- Map 23 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (CUSTOM_SIMPLE_EDGE), Reducer 28 (CUSTOM_SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Reducer 19 (CUSTOM_SIMPLE_EDGE), Reducer 33 (CUSTOM_SIMPLE_EDGE) -Reducer 21 <- Reducer 20 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 29 (SIMPLE_EDGE) -Reducer 26 <- Map 30 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 32 <- Map 31 (SIMPLE_EDGE), Map 34 (SIMPLE_EDGE) -Reducer 33 <- Reducer 32 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Reducer 36 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 4 <- Reducer 21 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 41 <- Map 40 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE) -Reducer 42 <- Map 45 (SIMPLE_EDGE), Reducer 41 (SIMPLE_EDGE) -Reducer 43 <- Reducer 42 (SIMPLE_EDGE) -Reducer 47 <- Map 46 (SIMPLE_EDGE), Map 54 (SIMPLE_EDGE) -Reducer 48 <- Map 55 (SIMPLE_EDGE), Reducer 47 (SIMPLE_EDGE) -Reducer 49 <- Reducer 48 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (CUSTOM_SIMPLE_EDGE) -Reducer 51 <- Reducer 50 (CUSTOM_SIMPLE_EDGE), Reducer 60 (CUSTOM_SIMPLE_EDGE) -Reducer 52 <- Reducer 51 (CUSTOM_SIMPLE_EDGE), Reducer 65 (CUSTOM_SIMPLE_EDGE) -Reducer 53 <- Reducer 52 (SIMPLE_EDGE) -Reducer 57 <- Map 56 (SIMPLE_EDGE), Map 61 (SIMPLE_EDGE) -Reducer 58 <- Map 62 (SIMPLE_EDGE), Reducer 57 (SIMPLE_EDGE) -Reducer 59 <- Reducer 58 (SIMPLE_EDGE) -Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE) -Reducer 60 <- Reducer 59 (CUSTOM_SIMPLE_EDGE) -Reducer 64 <- Map 63 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE) -Reducer 65 <- Reducer 64 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (PARTITION_EDGE) +Vertex 19 <- Vertex 18 (PARTITION_EDGE), Vertex 28 (PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (PARTITION_EDGE), Vertex 33 (PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 43 (SORT_PARTITION_EDGE) +Vertex 38 <- Union 5 (CONTAINS), Vertex 37 (SORT_PARTITION_EDGE), Vertex 53 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 41 <- Vertex 40 (SORT_PARTITION_EDGE), Vertex 44 (SORT_PARTITION_EDGE) +Vertex 42 <- Vertex 41 (SORT_PARTITION_EDGE), Vertex 45 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE) +Vertex 47 <- Vertex 46 (SORT_PARTITION_EDGE), Vertex 54 (SORT_PARTITION_EDGE) +Vertex 48 <- Vertex 47 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (PARTITION_EDGE) +Vertex 51 <- Vertex 50 (PARTITION_EDGE), Vertex 60 (PARTITION_EDGE) +Vertex 52 <- Vertex 51 (PARTITION_EDGE), Vertex 65 (PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE) +Vertex 57 <- Vertex 56 (SORT_PARTITION_EDGE), Vertex 61 (SORT_PARTITION_EDGE) +Vertex 58 <- Vertex 57 (SORT_PARTITION_EDGE), Vertex 62 (SORT_PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (PARTITION_EDGE) +Vertex 60 <- Vertex 59 (PARTITION_EDGE) +Vertex 64 <- Vertex 63 (SORT_PARTITION_EDGE), Vertex 66 (SORT_PARTITION_EDGE) +Vertex 65 <- Vertex 64 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_258] - Limit [LIM_257] (rows=1 width=112) - Number of rows:100 - Group By Operator [GBY_255] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Union 5 [CUSTOM_SIMPLE_EDGE] - <-Reducer 38 [CONTAINS] - Reduce Output Operator [RS_254] - Group By Operator [GBY_253] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col0)"] - Select Operator [SEL_249] (rows=191667562 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_384] (rows=191667562 width=135) - Conds:RS_246._col2=RS_247._col0(Inner),Output:["_col3","_col4"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_246] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_378] (rows=174243235 width=135) - Conds:RS_243._col1=RS_244._col0(Inner),Output:["_col2","_col3","_col4"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_243] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_369] (rows=158402938 width=135) - Conds:RS_240._col0=RS_241._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_240] - PartitionCols:_col0 - Select Operator [SEL_127] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_346] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_bill_customer_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_125] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_quantity","ws_list_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_241] - PartitionCols:_col0 - Select Operator [SEL_130] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_347] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_128] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_244] - PartitionCols:_col0 - Group By Operator [GBY_156] (rows=58079562 width=88) - Output:["_col0"],keys:_col1 - Select Operator [SEL_152] (rows=116159124 width=88) - Output:["_col1"] - Filter Operator [FIL_151] (rows=116159124 width=88) - predicate:(_col3 > 4) - Select Operator [SEL_360] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_150] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 42 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0 - Group By Operator [GBY_148] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_146] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_371] (rows=696954748 width=88) - Conds:RS_143._col1=RS_144._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 45 [SIMPLE_EDGE] - SHUFFLE [RS_144] - PartitionCols:_col0 - Select Operator [SEL_139] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_350] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_137] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 41 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_370] (rows=633595212 width=88) - Conds:RS_140._col0=RS_141._col0(Inner),Output:["_col1","_col3"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_140] - PartitionCols:_col0 - Select Operator [SEL_133] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_348] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_131] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 44 [SIMPLE_EDGE] - SHUFFLE [RS_141] - PartitionCols:_col0 - Select Operator [SEL_136] (rows=36525 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_349] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_134] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"] - <-Reducer 53 [SIMPLE_EDGE] - SHUFFLE [RS_247] - PartitionCols:_col0 - Group By Operator [GBY_238] (rows=52799601 width=322) - Output:["_col0"],keys:KEY._col0 - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_237] - PartitionCols:_col0 - Group By Operator [GBY_236] (rows=105599202 width=322) - Output:["_col0"],keys:_col2 - Select Operator [SEL_235] (rows=105599202 width=322) - Output:["_col2"] - Filter Operator [FIL_234] (rows=105599202 width=322) - predicate:(_col3 > (0.95 * _col1)) - Merge Join Operator [MERGEJOIN_383] (rows=316797606 width=322) - Conds:(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 51 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_231] - Merge Join Operator [MERGEJOIN_382] (rows=1 width=233) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 50 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_228] - Select Operator [SEL_186] (rows=1 width=8) - Filter Operator [FIL_185] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_183] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 49 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_182] - Group By Operator [GBY_181] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_178] (rows=348477374 width=88) - Group By Operator [GBY_177] (rows=348477374 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 48 [SIMPLE_EDGE] - SHUFFLE [RS_176] - PartitionCols:_col0 - Group By Operator [GBY_175] (rows=696954748 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_173] (rows=696954748 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_373] (rows=696954748 width=88) - Conds:RS_170._col1=RS_171._col0(Inner),Output:["_col6"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_171] - PartitionCols:_col0 - Select Operator [SEL_166] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_353] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_164] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 47 [SIMPLE_EDGE] - SHUFFLE [RS_170] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_372] (rows=633595212 width=88) - Conds:RS_167._col0=RS_168._col0(Inner),Output:["_col1"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_167] - PartitionCols:_col0 - Select Operator [SEL_160] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_351] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_158] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 54 [SIMPLE_EDGE] - SHUFFLE [RS_168] - PartitionCols:_col0 - Select Operator [SEL_163] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_352] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_161] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 60 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_229] - Group By Operator [GBY_211] (rows=1 width=224) - Output:["_col0"],aggregations:["max(VALUE._col0)"] - <-Reducer 59 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_210] - Group By Operator [GBY_209] (rows=1 width=224) - Output:["_col0"],aggregations:["max(_col1)"] - Select Operator [SEL_207] (rows=348477374 width=88) - Output:["_col1"] - Group By Operator [GBY_206] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 58 [SIMPLE_EDGE] - SHUFFLE [RS_205] - PartitionCols:_col0 - Group By Operator [GBY_204] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_202] (rows=696954748 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_375] (rows=696954748 width=88) - Conds:RS_199._col1=RS_200._col0(Inner),Output:["_col2","_col3","_col6"] - <-Map 62 [SIMPLE_EDGE] - SHUFFLE [RS_200] - PartitionCols:_col0 - Select Operator [SEL_195] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_356] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_193] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 57 [SIMPLE_EDGE] - SHUFFLE [RS_199] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_374] (rows=633595212 width=88) - Conds:RS_196._col0=RS_197._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_196] - PartitionCols:_col0 - Select Operator [SEL_189] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_354] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_187] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 61 [SIMPLE_EDGE] - SHUFFLE [RS_197] - PartitionCols:_col0 - Select Operator [SEL_192] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_355] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_190] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 65 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_232] - Group By Operator [GBY_226] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 64 [SIMPLE_EDGE] - SHUFFLE [RS_225] - PartitionCols:_col0 - Group By Operator [GBY_224] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_222] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_376] (rows=633595212 width=88) - Conds:RS_219._col0=RS_220._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_219] - PartitionCols:_col0 - Select Operator [SEL_215] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_357] (rows=575995635 width=88) - predicate:ss_customer_sk is not null - TableScan [TS_213] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 66 [SIMPLE_EDGE] - SHUFFLE [RS_220] - PartitionCols:_col0 - Select Operator [SEL_218] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_358] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_216] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_254] - Group By Operator [GBY_253] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col0)"] - Select Operator [SEL_124] (rows=383314495 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_381] (rows=383314495 width=135) - Conds:RS_121._col1=RS_122._col0(Inner),Output:["_col3","_col4"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Group By Operator [GBY_113] (rows=52799601 width=322) - Output:["_col0"],keys:KEY._col0 - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Group By Operator [GBY_111] (rows=105599202 width=322) - Output:["_col0"],keys:_col2 - Select Operator [SEL_110] (rows=105599202 width=322) - Output:["_col2"] - Filter Operator [FIL_109] (rows=105599202 width=322) - predicate:(_col3 > (0.95 * _col1)) - Merge Join Operator [MERGEJOIN_380] (rows=316797606 width=322) - Conds:(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_106] - Merge Join Operator [MERGEJOIN_379] (rows=1 width=233) - Conds:(Left Outer),Output:["_col1"] - <-Reducer 18 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_103] - Select Operator [SEL_61] (rows=1 width=8) - Filter Operator [FIL_60] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_58] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 17 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_57] - Group By Operator [GBY_56] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_53] (rows=348477374 width=88) - Group By Operator [GBY_52] (rows=348477374 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Group By Operator [GBY_50] (rows=696954748 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_48] (rows=696954748 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_365] (rows=696954748 width=88) - Conds:RS_45._col1=RS_46._col0(Inner),Output:["_col6"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_340] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_39] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_364] (rows=633595212 width=88) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_338] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_33] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_339] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_36] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 28 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_104] - Group By Operator [GBY_86] (rows=1 width=224) - Output:["_col0"],aggregations:["max(VALUE._col0)"] - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_85] - Group By Operator [GBY_84] (rows=1 width=224) - Output:["_col0"],aggregations:["max(_col1)"] - Select Operator [SEL_82] (rows=348477374 width=88) - Output:["_col1"] - Group By Operator [GBY_81] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Group By Operator [GBY_79] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_77] (rows=696954748 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_367] (rows=696954748 width=88) - Conds:RS_74._col1=RS_75._col0(Inner),Output:["_col2","_col3","_col6"] - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Select Operator [SEL_70] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_343] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_68] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_366] (rows=633595212 width=88) - Conds:RS_71._col0=RS_72._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Select Operator [SEL_64] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_341] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_62] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col0 - Select Operator [SEL_67] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_342] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_65] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 33 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_107] - Group By Operator [GBY_101] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_99] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_97] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_368] (rows=633595212 width=88) - Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_90] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_344] (rows=575995635 width=88) - predicate:ss_customer_sk is not null - TableScan [TS_88] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_93] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_345] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_91] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_377] (rows=348467716 width=135) - Conds:RS_118._col2=RS_119._col0(Inner),Output:["_col1","_col3","_col4"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=58079562 width=88) - Output:["_col0"],keys:_col1 - Select Operator [SEL_27] (rows=116159124 width=88) - Output:["_col1"] - Filter Operator [FIL_26] (rows=116159124 width=88) - predicate:(_col3 > 4) - Select Operator [SEL_359] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_25] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_21] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_363] (rows=696954748 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_337] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_362] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=36525 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_336] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_335] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_361] (rows=316788826 width=135) - Conds:RS_115._col0=RS_116._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_333] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_bill_customer_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity","cs_list_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_334] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query25.q.out ql/src/test/results/clientpositive/perf/query25.q.out index 8e5119c..fa6b185 100644 --- ql/src/test/results/clientpositive/perf/query25.q.out +++ ql/src/test/results/clientpositive/perf/query25.q.out @@ -5,141 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col18)","sum(_col23)","sum(_col3)"],keys:_col28, _col29, _col8, _col9 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col28","_col29","_col8","_col9","_col18","_col23","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col14=RS_44._col0(Inner),Output:["_col3","_col8","_col9","_col18","_col23","_col28","_col29"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col14 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col14, _col13(Inner),Output:["_col3","_col8","_col9","_col14","_col18","_col23"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col14, _col13 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col13","_col14","_col16","_col2","_col7"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col18","_col19"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_moy = 4) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_net_loss"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_net_profit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_net_profit"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query26.q.out ql/src/test/results/clientpositive/perf/query26.q.out index 6f7aaf7..73205b6 100644 --- ql/src/test/results/clientpositive/perf/query26.q.out +++ ql/src/test/results/clientpositive/perf/query26.q.out @@ -5,94 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_33] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_30] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(_col4)","avg(_col5)","avg(_col7)","avg(_col6)"],keys:_col18 - Select Operator [SEL_27] (rows=421645953 width=135) - Output:["_col18","_col4","_col5","_col7","_col6"] - Merge Join Operator [MERGEJOIN_58] (rows=421645953 width=135) - Conds:RS_24._col2=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col18"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=383314495 width=135) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col2","_col4","_col5","_col6","_col7"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_53] (rows=2300 width=1179) - predicate:(((p_channel_email = 'N') or (p_channel_event = 'N')) and p_promo_sk is not null) - TableScan [TS_9] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_email","p_channel_event"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_56] (rows=348467716 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_52] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=316788826 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_50] (rows=287989836 width=135) - predicate:(cs_bill_cdemo_sk is not null and cs_sold_date_sk is not null and cs_item_sk is not null and cs_promo_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_cdemo_sk","cs_item_sk","cs_promo_sk","cs_quantity","cs_list_price","cs_sales_price","cs_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_51] (rows=232725 width=385) - predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query27.q.out ql/src/test/results/clientpositive/perf/query27.q.out index cbd7d29..fe38aa9 100644 --- ql/src/test/results/clientpositive/perf/query27.q.out +++ ql/src/test/results/clientpositive/perf/query27.q.out @@ -45,96 +45,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Select Operator [SEL_32] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_31] (rows=1264972921 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_29] (rows=2529945843 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["avg(_col2)","avg(_col3)","avg(_col4)","avg(_col5)"],keys:_col0, _col1, 0 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_59] (rows=843315281 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col15","_col17"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_55] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_58] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col15"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=852 width=1910) - predicate:((s_state) IN ('SD', 'FL', 'MI', 'LA', 'MO', 'SC') and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_57] (rows=696954748 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_53] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_56] (rows=633595212 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_51] (rows=575995635 width=88) - predicate:(ss_cdemo_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_cdemo_sk","ss_store_sk","ss_quantity","ss_list_price","ss_sales_price","ss_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_52] (rows=232725 width=385) - predicate:((cd_gender = 'M') and (cd_marital_status = 'U') and (cd_education_status = '2 yr Degree') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query28.q.out ql/src/test/results/clientpositive/perf/query28.q.out index f7c5225..9c6aa18 100644 --- ql/src/test/results/clientpositive/perf/query28.q.out +++ ql/src/test/results/clientpositive/perf/query28.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Vertex 3' is a cross product PREHOOK: query: explain select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT @@ -104,108 +104,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (PARTITION_EDGE), Vertex 13 (PARTITION_EDGE), Vertex 2 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE), Vertex 7 (PARTITION_EDGE), Vertex 9 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_51] - Limit [LIM_50] (rows=1 width=2497) - Number of rows:100 - Select Operator [SEL_49] (rows=1 width=2497) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Merge Join Operator [MERGEJOIN_58] (rows=1 width=2497) - Conds:(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - <-Reducer 11 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_46] - Group By Operator [GBY_33] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_31] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_30] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_56] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 11 AND 15 and (ss_list_price BETWEEN 66 AND 76 or ss_coupon_amt BETWEEN 920 AND 1920 or ss_wholesale_cost BETWEEN 4 AND 24)) - TableScan [TS_28] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 13 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_47] - Group By Operator [GBY_40] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_39] - Group By Operator [GBY_38] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_37] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_57] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 6 AND 10 and (ss_list_price BETWEEN 91 AND 101 or ss_coupon_amt BETWEEN 1430 AND 2430 or ss_wholesale_cost BETWEEN 32 AND 52)) - TableScan [TS_35] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_42] - Group By Operator [GBY_5] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_4] - Group By Operator [GBY_3] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_2] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_52] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 0 AND 5 and (ss_list_price BETWEEN 11 AND 21 or ss_coupon_amt BETWEEN 460 AND 1460 or ss_wholesale_cost BETWEEN 14 AND 34)) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_43] - Group By Operator [GBY_12] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_9] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_53] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 26 AND 30 and (ss_list_price BETWEEN 28 AND 38 or ss_coupon_amt BETWEEN 2513 AND 3513 or ss_wholesale_cost BETWEEN 42 AND 62)) - TableScan [TS_7] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 7 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_44] - Group By Operator [GBY_19] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_18] - Group By Operator [GBY_17] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_16] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_54] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 21 AND 25 and (ss_list_price BETWEEN 135 AND 145 or ss_coupon_amt BETWEEN 14180 AND 15180 or ss_wholesale_cost BETWEEN 38 AND 58)) - TableScan [TS_14] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Reducer 9 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_45] - Group By Operator [GBY_26] (rows=1 width=416) - Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_24] (rows=21333171 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price - Select Operator [SEL_23] (rows=21333171 width=88) - Output:["ss_list_price"] - Filter Operator [FIL_55] (rows=21333171 width=88) - predicate:(ss_quantity BETWEEN 16 AND 20 and (ss_list_price BETWEEN 142 AND 152 or ss_coupon_amt BETWEEN 3054 AND 4054 or ss_wholesale_cost BETWEEN 80 AND 100)) - TableScan [TS_21] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query29.q.out ql/src/test/results/clientpositive/perf/query29.q.out index 587c027..f1fe37a 100644 --- ql/src/test/results/clientpositive/perf/query29.q.out +++ ql/src/test/results/clientpositive/perf/query29.q.out @@ -5,141 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 13 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_54] - Limit [LIM_53] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_52] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Group By Operator [GBY_49] (rows=510205767 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_47] (rows=1020411534 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col17)","sum(_col22)","sum(_col3)"],keys:_col27, _col28, _col7, _col8 - Select Operator [SEL_46] (rows=1020411534 width=88) - Output:["_col27","_col28","_col7","_col8","_col17","_col22","_col3"] - Merge Join Operator [MERGEJOIN_100] (rows=1020411534 width=88) - Conds:RS_43._col13=RS_44._col0(Inner),Output:["_col3","_col7","_col8","_col17","_col22","_col27","_col28"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_93] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col13 - Merge Join Operator [MERGEJOIN_99] (rows=927646829 width=88) - Conds:RS_40._col1, _col2=RS_41._col14, _col13(Inner),Output:["_col3","_col7","_col8","_col13","_col17","_col22"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col14, _col13 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col11","_col13","_col14","_col16","_col2","_col7"] - Merge Join Operator [MERGEJOIN_98] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col5","_col7","_col8","_col10","_col18","_col19"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_92] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_18] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_97] (rows=766650239 width=88) - Conds:RS_27._col6=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col8","_col10"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_91] (rows=4058 width=1119) - predicate:(d_moy BETWEEN 2 AND 5 and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_96] (rows=696954748 width=88) - Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_moy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_95] (rows=633595212 width=88) - Conds:RS_21._col1, _col2, _col4=RS_22._col1, _col2, _col3(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col8","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_11] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_89] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_9] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_return_quantity"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_88] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_quantity"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_94] (rows=316788826 width=135) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_86] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_87] (rows=36525 width=1119) - predicate:((d_year) IN (2000, 2001, 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query3.q.out ql/src/test/results/clientpositive/perf/query3.q.out index 2845a88..acddbe0 100644 --- ql/src/test/results/clientpositive/perf/query3.q.out +++ ql/src/test/results/clientpositive/perf/query3.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col8, _col4, _col5 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col8","_col4","_col5","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col2","_col4","_col5","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=36524 width=1119) - predicate:((d_moy = 12) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col2","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=231000 width=1436) - predicate:((i_manufact_id = 436) and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manufact_id"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query30.q.out ql/src/test/results/clientpositive/perf/query30.q.out index 8d38fe2..ca187c4 100644 --- ql/src/test/results/clientpositive/perf/query30.q.out +++ ql/src/test/results/clientpositive/perf/query30.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 22' is a cross product +Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 22' is a cross product PREHOOK: query: explain with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, @@ -60,249 +60,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (CUSTOM_SIMPLE_EDGE), Map 30 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE) -Reducer 26 <- Map 29 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 18 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_102] - Limit [LIM_101] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_100] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_99] - Select Operator [SEL_98] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - Filter Operator [FIL_97] (rows=930023387364950016 width=1) - predicate:(_col2 > CASE WHEN (_col20 is null) THEN (null) ELSE (_col19) END) - Select Operator [SEL_96] (rows=2790070162094850048 width=1) - Output:["_col2","_col6","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - Merge Join Operator [MERGEJOIN_162] (rows=2790070162094850048 width=1) - Conds:RS_93._col17=RS_94._col2(Left Outer),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col18","_col19","_col20"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col2 - Select Operator [SEL_86] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_85] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_84] - PartitionCols:_col0 - Group By Operator [GBY_83] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col3 - Select Operator [SEL_82] (rows=5072854730221289472 width=1) - Output:["_col3","_col2"] - Merge Join Operator [MERGEJOIN_161] (rows=5072854730221289472 width=1) - Conds:RS_79._col1=RS_80._col0(Inner),Output:["_col2","_col3"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=22000000 width=1014) - Output:["_col1","_col2"] - Group By Operator [GBY_44] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1 - Group By Operator [GBY_42] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col7, _col1 - Select Operator [SEL_41] (rows=44000000 width=1014) - Output:["_col7","_col1","_col3"] - Merge Join Operator [MERGEJOIN_159] (rows=44000000 width=1014) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_32] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_158] (rows=15838314 width=92) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_146] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_returning_addr_sk is not null) - TableScan [TS_26] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_returning_customer_sk","wr_returning_addr_sk","wr_return_amt"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_147] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Group By Operator [GBY_77] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Group By Operator [GBY_75] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_154] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_70] - Select Operator [SEL_47] (rows=80000000 width=4) - TableScan [TS_46] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:COMPLETE - <-Map 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_72] - Select Operator [SEL_69] (rows=40000000 width=4) - TableScan [TS_68] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_67] (rows=22000000 width=1014) - Output:["_col1"] - Group By Operator [GBY_66] (rows=22000000 width=1014) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Group By Operator [GBY_64] (rows=44000000 width=1014) - Output:["_col0","_col1"],keys:_col7, _col1 - Select Operator [SEL_63] (rows=44000000 width=1014) - Output:["_col7","_col1"] - Merge Join Operator [MERGEJOIN_153] (rows=44000000 width=1014) - Conds:RS_60._col2=RS_61._col0(Inner),Output:["_col1","_col7"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_56] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_54] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_152] (rows=15838314 width=92) - Conds:RS_57._col0=RS_58._col0(Inner),Output:["_col1","_col2"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=14398467 width=92) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_149] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_returning_addr_sk is not null) - TableScan [TS_48] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_returning_customer_sk","wr_returning_addr_sk"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_150] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_51] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col17 - Merge Join Operator [MERGEJOIN_160] (rows=96800003 width=860) - Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col17","_col18"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1 - Group By Operator [GBY_22] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col1, _col7 - Select Operator [SEL_21] (rows=44000000 width=1014) - Output:["_col1","_col7","_col3"] - Merge Join Operator [MERGEJOIN_157] (rows=44000000 width=1014) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_12] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_156] (rows=15838314 width=92) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_144] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_143] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_returning_addr_sk is not null and wr_returning_customer_sk is not null) - TableScan [TS_6] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_returning_customer_sk","wr_returning_addr_sk","wr_return_amt"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_155] (rows=88000001 width=860) - Conds:RS_87._col2=RS_88._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"] - Filter Operator [FIL_141] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_current_addr_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag","c_birth_day","c_birth_month","c_birth_year","c_birth_country","c_login","c_email_address","c_last_review_date"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_142] (rows=20000000 width=1014) - predicate:((ca_state = 'IL') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query31.q.out ql/src/test/results/clientpositive/perf/query31.q.out index 433d419..4ea7e20 100644 --- ql/src/test/results/clientpositive/perf/query31.q.out +++ ql/src/test/results/clientpositive/perf/query31.q.out @@ -5,333 +5,32 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 18 <- Map 21 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) -Reducer 24 <- Map 28 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 25 <- Reducer 24 (SIMPLE_EDGE) -Reducer 26 <- Reducer 25 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 29 (SIMPLE_EDGE), Map 33 (SIMPLE_EDGE) -Reducer 31 <- Map 34 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Reducer 31 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Map 40 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 38 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 38 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_140] - Select Operator [SEL_139] (rows=316243230 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_138] - Select Operator [SEL_137] (rows=316243230 width=88) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_136] (rows=316243230 width=88) - predicate:CASE WHEN ((_col3 > 0)) THEN (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > (_col5 / _col3))) ELSE ((null > (_col5 / _col3))) END) ELSE (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > null)) ELSE (null) END) END - Merge Join Operator [MERGEJOIN_272] (rows=632486460 width=88) - Conds:RS_132._col6=RS_133._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col7","_col9","_col11"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Group By Operator [GBY_130] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col0 - Group By Operator [GBY_128] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_127] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_269] (rows=174243235 width=135) - Conds:RS_124._col1=RS_125._col0(Inner),Output:["_col2","_col7"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_125] - PartitionCols:_col0 - Select Operator [SEL_120] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_257] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_118] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_124] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_268] (rows=158402938 width=135) - Conds:RS_121._col0=RS_122._col0(Inner),Output:["_col1","_col2"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col0 - Select Operator [SEL_114] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_255] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_112] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Select Operator [SEL_117] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_256] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_115] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col6 - Filter Operator [FIL_110] (rows=574987679 width=88) - predicate:CASE WHEN ((_col1 > 0)) THEN (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > (_col3 / _col1))) ELSE ((null > (_col3 / _col1))) END) ELSE (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > null)) ELSE (null) END) END - Select Operator [SEL_109] (rows=1149975359 width=88) - Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_271] (rows=1149975359 width=88) - Conds:RS_104._col0=RS_105._col0(Inner),RS_104._col0=RS_106._col0(Inner),RS_104._col0=RS_107._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Group By Operator [GBY_36] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_35] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_261] (rows=696954748 width=88) - Conds:RS_32._col1=RS_33._col0(Inner),Output:["_col2","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_245] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_26] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_260] (rows=633595212 width=88) - Conds:RS_29._col0=RS_30._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_243] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_20] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_244] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col0 - Group By Operator [GBY_58] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Group By Operator [GBY_56] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_55] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_263] (rows=696954748 width=88) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col7"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_248] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_46] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_262] (rows=633595212 width=88) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_246] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_40] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_45] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_247] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_43] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_270] (rows=95833780 width=135) - Conds:RS_100._col0=RS_101._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_78] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Group By Operator [GBY_76] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_75] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_265] (rows=174243235 width=135) - Conds:RS_72._col1=RS_73._col0(Inner),Output:["_col2","_col7"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_68] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_251] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_66] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_264] (rows=158402938 width=135) - Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col0 - Select Operator [SEL_62] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_249] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_60] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_65] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_250] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_63] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Group By Operator [GBY_98] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Group By Operator [GBY_96] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_95] (rows=174243235 width=135) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_267] (rows=174243235 width=135) - Conds:RS_92._col1=RS_93._col0(Inner),Output:["_col2","_col7"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_254] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_86] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_266] (rows=158402938 width=135) - Conds:RS_89._col0=RS_90._col0(Inner),Output:["_col1","_col2"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Select Operator [SEL_82] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_252] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_80] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_85] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_253] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_83] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col2"] - Merge Join Operator [MERGEJOIN_259] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_242] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_258] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_240] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_241] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query32.q.out ql/src/test/results/clientpositive/perf/query32.q.out index 6311054..5304734 100644 --- ql/src/test/results/clientpositive/perf/query32.q.out +++ ql/src/test/results/clientpositive/perf/query32.q.out @@ -39,96 +39,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Map 10 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 9 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_38] - Group By Operator [GBY_36] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_35] - Group By Operator [GBY_34] (rows=1 width=112) - Output:["_col0"],aggregations:["sum(_col1)"] - Select Operator [SEL_33] (rows=232311810 width=135) - Output:["_col1"] - Filter Operator [FIL_32] (rows=232311810 width=135) - predicate:(_col1 > _col5) - Select Operator [SEL_31] (rows=696935432 width=135) - Output:["_col1","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=696935432 width=135) - Conds:RS_27._col0=RS_28._col0(Inner),RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col5"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=231000 width=1436) - Output:["_col0"] - Filter Operator [FIL_55] (rows=231000 width=1436) - predicate:((i_manufact_id = 436) and i_item_sk is not null) - TableScan [TS_14] (rows=462000 width=1436) - default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=158394413 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_12] (rows=158394413 width=135) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=316788826 width=135) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Merge Join Operator [MERGEJOIN_58] (rows=316788826 width=135) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_53] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2000-01-27' AND '2000-04-27' and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=316788826 width=135) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_59] (rows=316788826 width=135) - Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_57] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2000-01-27' AND '2000-04-27' and d_date_sk is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_56] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null) - TableScan [TS_17] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query33.q.out ql/src/test/results/clientpositive/perf/query33.q.out index 342bd90..e6cb0b3 100644 --- ql/src/test/results/clientpositive/perf/query33.q.out +++ ql/src/test/results/clientpositive/perf/query33.q.out @@ -147,289 +147,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query34.q.out ql/src/test/results/clientpositive/perf/query34.q.out index 51710bd..8200edd 100644 --- ql/src/test/results/clientpositive/perf/query34.q.out +++ ql/src/test/results/clientpositive/perf/query34.q.out @@ -5,96 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=88000001 width=860) - Conds:RS_31._col0=RS_32._col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Filter Operator [FIL_29] (rows=42591679 width=88) - predicate:_col2 BETWEEN 15 AND 20 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_56] (rows=852 width=1910) - predicate:((s_county) IN ('Kittitas County', 'Adams County', 'Richland County', 'Furnas County', 'Orange County', 'Appanoose County', 'Franklin Parish', 'Tehama County') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1200 width=107) - Output:["_col0"] - Filter Operator [FIL_55] (rows=1200 width=107) - predicate:(((hd_buy_potential = '1001-5000') or (hd_buy_potential = '5001-10000')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.2)) ELSE (null) END and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential","hd_dep_count","hd_vehicle_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_store_sk","ss_ticket_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=8116 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and (d_dom BETWEEN 1 AND 3 or d_dom BETWEEN 25 AND 28) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query36.q.out ql/src/test/results/clientpositive/perf/query36.q.out index b356628..df4c901 100644 --- ql/src/test/results/clientpositive/perf/query36.q.out +++ ql/src/test/results/clientpositive/perf/query36.q.out @@ -59,91 +59,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_33] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Select Operator [SEL_30] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_29] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(_col4 / _col5) ASC NULLS FIRST","partition by:":"(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((UDFToInteger(grouping(_col6, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_28] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((UDFToInteger(grouping(_col6, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_26] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5","_col6"] - Group By Operator [GBY_25] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_23] (rows=2299950717 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1, 0 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_51] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col4","_col10","_col11"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_48] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_50] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_47] (rows=852 width=1910) - predicate:((s_state) IN ('SD', 'FL', 'MI', 'LA', 'MO', 'SC', 'AL', 'GA') and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_49] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_45] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_ext_sales_price","ss_net_profit"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_46] (rows=36524 width=1119) - predicate:((d_year = 1999) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query37.q.out ql/src/test/results/clientpositive/perf/query37.q.out index 1b61198..bca5d0e 100644 --- ql/src/test/results/clientpositive/perf/query37.q.out +++ ql/src/test/results/clientpositive/perf/query37.q.out @@ -33,71 +33,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_26] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_23] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_21] (rows=633577652 width=135) - Output:["_col0","_col1","_col2"],keys:_col2, _col3, _col4 - Merge Join Operator [MERGEJOIN_44] (rows=633577652 width=135) - Conds:RS_16._col0=RS_17._col0(Inner),RS_17._col0=RS_18._col1(Inner),Output:["_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0"] - Filter Operator [FIL_39] (rows=287989836 width=135) - predicate:cs_item_sk is not null - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25666 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=25666 width=1436) - predicate:((i_manufact_id) IN (678, 964, 918, 849) and i_current_price BETWEEN 22 AND 52 and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_manufact_id"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=4593600 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4176000 width=15) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=4176000 width=15) - predicate:(inv_quantity_on_hand BETWEEN 100 AND 500 and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_6] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN 2001-06-02 AND 2001-08-01 00:00:00.0 and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query38.q.out ql/src/test/results/clientpositive/perf/query38.q.out index 1db8b6a..476830b 100644 --- ql/src/test/results/clientpositive/perf/query38.q.out +++ ql/src/test/results/clientpositive/perf/query38.q.out @@ -45,188 +45,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 18 <- Map 21 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 5 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 5 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_92] - Limit [LIM_91] (rows=1 width=16) - Number of rows:100 - Group By Operator [GBY_89] (rows=1 width=16) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_88] - Group By Operator [GBY_87] (rows=1 width=16) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_85] (rows=1 width=108) - Filter Operator [FIL_84] (rows=1 width=108) - predicate:(_col3 = 3) - Select Operator [SEL_117] (rows=152458212 width=108) - Output:["_col3"] - Group By Operator [GBY_83] (rows=152458212 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_50] (rows=87116929 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_44] (rows=174233858 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_42] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_41] (rows=348467716 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_121] (rows=348467716 width=135) - Conds:RS_38._col1=RS_39._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_113] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_32] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_120] (rows=316788826 width=135) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_111] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_26] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_112] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_77] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_71] (rows=87121617 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_69] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_68] (rows=174243235 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_123] (rows=174243235 width=135) - Conds:RS_65._col1=RS_66._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Select Operator [SEL_61] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_116] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_59] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_122] (rows=158402938 width=135) - Conds:RS_62._col0=RS_63._col0(Inner),Output:["_col1","_col3"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_114] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_53] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Select Operator [SEL_58] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_115] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_56] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_24] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_119] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_110] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_118] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_108] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_customer_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_109] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query39.q.out ql/src/test/results/clientpositive/perf/query39.q.out index bde20a6..3b83861 100644 --- ql/src/test/results/clientpositive/perf/query39.q.out +++ ql/src/test/results/clientpositive/perf/query39.q.out @@ -5,163 +5,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_62] - Select Operator [SEL_61] (rows=13756683 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Select Operator [SEL_59] (rows=13756683 width=15) - Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col8","_col9"] - Merge Join Operator [MERGEJOIN_103] (rows=13756683 width=15) - Conds:RS_56._col1, _col2=RS_57._col1, _col2(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col1, _col2 - Select Operator [SEL_55] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_54] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_53] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_52] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_50] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Select Operator [SEL_49] (rows=50024305 width=15) - Output:["_col8","_col7","_col9","_col3"] - Merge Join Operator [MERGEJOIN_102] (rows=50024305 width=15) - Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_37] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_101] (rows=45476640 width=15) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_95] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_34] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_100] (rows=41342400 width=15) - Conds:RS_40._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_30] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_93] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_28] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_33] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_94] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 4) and d_date_sk is not null) - TableScan [TS_31] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col1, _col2 - Select Operator [SEL_27] (rows=12506076 width=15) - Output:["_col1","_col2","_col3","_col4"] - Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END - Select Operator [SEL_25] (rows=25012152 width=15) - Output:["_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_22] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col8, _col7, _col9 - Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col8","_col7","_col9","_col3"] - Merge Join Operator [MERGEJOIN_99] (rows=50024305 width=15) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col7","_col8","_col9"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_92] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_9] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_98] (rows=45476640 width=15) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_91] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_97] (rows=41342400 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_89] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_0] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_90] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 7 PREHOOK: query: with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =1999 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=3 and inv2.d_moy=3+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/perf/query40.q.out ql/src/test/results/clientpositive/perf/query40.q.out index a45fdaf..1caa2c8 100644 --- ql/src/test/results/clientpositive/perf/query40.q.out +++ ql/src/test/results/clientpositive/perf/query40.q.out @@ -55,94 +55,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_34] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Group By Operator [GBY_31] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1 - Group By Operator [GBY_29] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)","sum(_col3)"],keys:_col0, _col1 - Select Operator [SEL_27] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_58] (rows=421645953 width=135) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col7","_col9","_col11","_col14"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_12] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_state"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_57] (rows=383314495 width=135) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col4","_col7","_col9","_col11"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=51333 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_53] (rows=51333 width=1436) - predicate:(i_current_price BETWEEN 0.99 AND 1.49 and i_item_sk is not null) - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_current_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_56] (rows=348467716 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2","_col4","_col7","_col9"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_52] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-03-08 23:00:00.0 AND 1998-05-08 00:00:00.0 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=316788826 width=135) - Conds:RS_15._col2, _col3=RS_16._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2, _col3 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_50] (rows=287989836 width=135) - predicate:(cs_warehouse_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_warehouse_sk","cs_item_sk","cs_order_number","cs_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_51] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_3] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query42.q.out ql/src/test/results/clientpositive/perf/query42.q.out index 7100bb3..58e1465 100644 --- ql/src/test/results/clientpositive/perf/query42.q.out +++ ql/src/test/results/clientpositive/perf/query42.q.out @@ -5,66 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_23] - Limit [LIM_22] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_21] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_33] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_31] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category_id","i_category","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_32] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_29] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_30] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query43.q.out ql/src/test/results/clientpositive/perf/query43.q.out index 72712e8..846ab70 100644 --- ql/src/test/results/clientpositive/perf/query43.q.out +++ ql/src/test/results/clientpositive/perf/query43.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col0, _col1 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col5","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=852 width=1910) - predicate:((s_gmt_offset = -6) and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name","s_gmt_offset"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_31] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_day_name"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query45.q.out ql/src/test/results/clientpositive/perf/query45.q.out index e2d0da5..0c601f8 100644 --- ql/src/test/results/clientpositive/perf/query45.q.out +++ ql/src/test/results/clientpositive/perf/query45.q.out @@ -5,117 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 9 <- Reducer 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_47] - Limit [LIM_46] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_45] (rows=95833781 width=135) - Output:["_col0","_col1","_col2"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_44] - Group By Operator [GBY_42] (rows=95833781 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Group By Operator [GBY_40] (rows=191667562 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col11)"],keys:_col4, _col3 - Select Operator [SEL_39] (rows=191667562 width=135) - Output:["_col4","_col3","_col11"] - Merge Join Operator [MERGEJOIN_74] (rows=191667562 width=135) - Conds:RS_36._col0=RS_37._col5(Inner),Output:["_col3","_col4","_col11"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_70] (rows=88000001 width=860) - Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col0","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_64] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_65] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_zip"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_73] (rows=174243235 width=135) - Conds:RS_29._col0=RS_30._col1(Inner),Output:["_col5","_col6"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_72] (rows=158402938 width=135) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_68] (rows=144002668 width=135) - predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null and ws_item_sk is not null) - TableScan [TS_16] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk","ws_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_69] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_19] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_71] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_66] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_14] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_11] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_67] (rows=231000 width=1436) - predicate:(i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) - TableScan [TS_9] (rows=462000 width=1436) - default@item,i2,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query46.q.out ql/src/test/results/clientpositive/perf/query46.q.out index d0275c6..78399d5 100644 --- ql/src/test/results/clientpositive/perf/query46.q.out +++ ql/src/test/results/clientpositive/perf/query46.q.out @@ -5,130 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 15 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 12 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 8 <- Map 13 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_50] - Limit [LIM_49] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_48] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Select Operator [SEL_46] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_45] (rows=463823414 width=88) - predicate:(_col5 <> _col8) - Merge Join Operator [MERGEJOIN_86] (rows=463823414 width=88) - Conds:RS_42._col0=RS_43._col1(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9","_col10"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_37] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_36] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_34] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col6)","sum(_col7)"],keys:_col1, _col17, _col3, _col5 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col17","_col3","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_85] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col17"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_18] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=766650239 width=88) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=7200 width=107) - Output:["_col0"] - Filter Operator [FIL_79] (rows=7200 width=107) - predicate:(((hd_dep_count = 4) or (hd_vehicle_count = 2)) and hd_demo_sk is not null) - TableScan [TS_15] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=696954748 width=88) - Conds:RS_24._col4=RS_25._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_78] (rows=852 width=1910) - predicate:((s_city) IN ('Rosedale', 'Bethlehem', 'Clinton', 'Clifton', 'Springfield') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_city"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_82] (rows=633595212 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=18263 width=1119) - Output:["_col0"] - Filter Operator [FIL_77] (rows=18263 width=1119) - predicate:((d_dow) IN (6, 0) and (d_year) IN (1998, 1999, 2000) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dow"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_76] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_customer_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_coupon_amt","ss_net_profit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=88000001 width=860) - Conds:RS_39._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_74] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,current_addr,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query48.q.out ql/src/test/results/clientpositive/perf/query48.q.out index 3b390f2..1eea1e7 100644 --- ql/src/test/results/clientpositive/perf/query48.q.out +++ ql/src/test/results/clientpositive/perf/query48.q.out @@ -5,90 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 6 <- Map 9 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_35] - Group By Operator [GBY_33] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_32] - Group By Operator [GBY_31] (rows=1 width=8) - Output:["_col0"],aggregations:["sum(_col6)"] - Merge Join Operator [MERGEJOIN_58] (rows=15616946 width=88) - Conds:RS_27._col0=RS_28._col0(Inner),Output:["_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_50] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=14197224 width=88) - Output:["_col0","_col4"] - Filter Operator [FIL_25] (rows=14197224 width=88) - predicate:(((_col12) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 0 AND 2000) or ((_col12) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 3000) or ((_col12) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 25000)) - Select Operator [SEL_24] (rows=85183356 width=88) - Output:["_col0","_col4","_col6","_col12"] - Merge Join Operator [MERGEJOIN_57] (rows=85183356 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col5","_col7","_col9"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=465450 width=385) - Output:["_col0"] - Filter Operator [FIL_54] (rows=465450 width=385) - predicate:((cd_marital_status = 'M') and (cd_education_status = '4 yr Degree') and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_56] (rows=77439413 width=88) - Conds:RS_18._col3=RS_19._col0(Inner),Output:["_col1","_col2","_col5","_col7","_col9"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_53] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_9] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_55] (rows=70399465 width=88) - Conds:RS_15._col0=RS_16._col3(Inner),Output:["_col1","_col2","_col3","_col5","_col7"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1704 width=1910) - Output:["_col0"] - Filter Operator [FIL_51] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_3] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col3 - Select Operator [SEL_8] (rows=63999513 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] - Filter Operator [FIL_52] (rows=63999513 width=88) - predicate:((ss_sales_price BETWEEN 100 AND 150 or ss_sales_price BETWEEN 50 AND 100 or ss_sales_price BETWEEN 150 AND 200) and (ss_net_profit BETWEEN 0 AND 2000 or ss_net_profit BETWEEN 150 AND 3000 or ss_net_profit BETWEEN 50 AND 25000) and ss_store_sk is not null and ss_cdemo_sk is not null and ss_addr_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_cdemo_sk","ss_addr_sk","ss_store_sk","ss_quantity","ss_sales_price","ss_net_profit"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query5.q.out ql/src/test/results/clientpositive/perf/query5.q.out index d3ec922..439dd6d 100644 --- ql/src/test/results/clientpositive/perf/query5.q.out +++ ql/src/test/results/clientpositive/perf/query5.q.out @@ -255,235 +255,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 17 <- Union 13 (CONTAINS) -Map 20 <- Union 21 (CONTAINS) -Map 9 <- Union 2 (CONTAINS) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Union 13 (SIMPLE_EDGE) -Reducer 15 <- Map 19 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 22 <- Map 28 (SIMPLE_EDGE), Union 21 (SIMPLE_EDGE) -Reducer 23 <- Map 29 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Map 25 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE), Union 21 (CONTAINS) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 6 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Union 13 (CONTAINS) +Vertex 20 <- Union 21 (CONTAINS) +Vertex 22 <- Union 21 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 24 <- Union 6 (CONTAINS), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 26 <- Union 21 (CONTAINS), Vertex 25 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_93] - Limit [LIM_92] (rows=100 width=110) - Number of rows:100 - Select Operator [SEL_91] (rows=1136898901 width=110) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_90] - Select Operator [SEL_89] (rows=1136898901 width=110) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_88] (rows=1136898901 width=110) - Output:["_col0","_col1","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 16 [CONTAINS] - Reduce Output Operator [RS_87] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_86] (rows=2273797803 width=110) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_49] (rows=191657181 width=132) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_48] (rows=191657181 width=132) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_46] (rows=383314363 width=132) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col4)","sum(_col3)","sum(_col5)"],keys:_col9 - Select Operator [SEL_45] (rows=383314363 width=132) - Output:["_col9","_col2","_col4","_col3","_col5"] - Merge Join Operator [MERGEJOIN_136] (rows=383314363 width=132) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=46000 width=460) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=46000 width=460) - predicate:cp_catalog_page_sk is not null - TableScan [TS_36] (rows=46000 width=460) - default@catalog_page,catalog_page,Tbl:COMPLETE,Col:NONE,Output:["cp_catalog_page_sk","cp_catalog_page_id"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_135] (rows=348467596 width=132) - Conds:Union 13._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_125] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - TableScan [TS_33] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_27] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_123] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_catalog_page_sk is not null) - TableScan [TS_25] (rows=287989836 width=135) - Output:["cs_sold_date_sk","cs_catalog_page_sk","cs_ext_sales_price","cs_net_profit"] - <-Map 17 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_30] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_124] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_catalog_page_sk is not null) - TableScan [TS_28] (rows=28798881 width=106) - Output:["cr_returned_date_sk","cr_catalog_page_sk","cr_return_amount","cr_net_loss"] - <-Reducer 24 [CONTAINS] - Reduce Output Operator [RS_87] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_86] (rows=2273797803 width=110) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_83] (rows=182955399 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_82] (rows=182955399 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0 - Group By Operator [GBY_80] (rows=365910798 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col4)","sum(_col3)","sum(_col5)"],keys:_col9 - Select Operator [SEL_79] (rows=365910798 width=135) - Output:["_col9","_col2","_col4","_col3","_col5"] - Merge Join Operator [MERGEJOIN_138] (rows=365910798 width=135) - Conds:RS_76._col0=RS_77._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=84 width=1850) - Output:["_col0","_col1"] - Filter Operator [FIL_131] (rows=84 width=1850) - predicate:web_site_sk is not null - TableScan [TS_70] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_site_id"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_137] (rows=332646173 width=135) - Conds:Union 21._col1=RS_74._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_130] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - TableScan [TS_67] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Union 21 [SIMPLE_EDGE] - <-Map 20 [CONTAINS] - Reduce Output Operator [RS_73] - PartitionCols:_col1 - Select Operator [SEL_54] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_127] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_web_site_sk is not null) - TableScan [TS_52] (rows=144002668 width=135) - Output:["ws_sold_date_sk","ws_web_site_sk","ws_ext_sales_price","ws_net_profit"] - <-Reducer 26 [CONTAINS] - Reduce Output Operator [RS_73] - PartitionCols:_col1 - Select Operator [SEL_64] (rows=158402938 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_132] (rows=158402938 width=135) - Conds:RS_61._col0, _col2=RS_62._col1, _col2(Inner),Output:["_col1","_col3","_col6","_col7"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0, _col2 - Select Operator [SEL_57] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_128] (rows=144002668 width=135) - predicate:(ws_web_site_sk is not null and ws_order_number is not null and ws_item_sk is not null) - TableScan [TS_55] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_item_sk","ws_web_site_sk","ws_order_number"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col1, _col2 - Select Operator [SEL_60] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_129] (rows=14398467 width=92) - predicate:(wr_returned_date_sk is not null and wr_item_sk is not null and wr_order_number is not null) - TableScan [TS_58] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_item_sk","wr_order_number","wr_return_amt","wr_net_loss"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_87] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_86] (rows=2273797803 width=110) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_24] (rows=383320021 width=87) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_23] (rows=383320021 width=87) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Group By Operator [GBY_21] (rows=766640042 width=87) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col2)","sum(_col4)","sum(_col3)","sum(_col5)"],keys:_col9 - Select Operator [SEL_20] (rows=766640042 width=87) - Output:["_col9","_col2","_col4","_col3","_col5"] - Merge Join Operator [MERGEJOIN_134] (rows=766640042 width=87) - Conds:RS_17._col0=RS_18._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_122] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_11] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_133] (rows=696945478 width=87) - Conds:Union 2._col1=RS_15._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_121] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - TableScan [TS_8] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_14] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_119] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - Output:["ss_sold_date_sk","ss_store_sk","ss_ext_sales_price","ss_net_profit"] - <-Map 9 [CONTAINS] - Reduce Output Operator [RS_14] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_120] (rows=57591150 width=77) - predicate:(sr_returned_date_sk is not null and sr_store_sk is not null) - TableScan [TS_3] (rows=57591150 width=77) - Output:["sr_returned_date_sk","sr_store_sk","sr_return_amt","sr_net_loss"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query50.q.out ql/src/test/results/clientpositive/perf/query50.q.out index 47a00b0..3507155 100644 --- ql/src/test/results/clientpositive/perf/query50.q.out +++ ql/src/test/results/clientpositive/perf/query50.q.out @@ -117,94 +117,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Group By Operator [GBY_31] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Group By Operator [GBY_29] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] - Merge Join Operator [MERGEJOIN_59] (rows=843315281 width=88) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col0","_col5","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1704 width=1910) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"] - Filter Operator [FIL_55] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_id","s_street_number","s_street_name","s_street_type","s_suite_number","s_city","s_county","s_state","s_zip"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=766650239 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=73049 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_57] (rows=696954748 width=88) - Conds:RS_18._col5=RS_19._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_53] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_56] (rows=633595212 width=88) - Conds:RS_15._col1, _col2, _col4=RS_16._col1, _col2, _col3(Inner),Output:["_col0","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1, _col2, _col4 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_51] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_store_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_52] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) - TableScan [TS_3] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query51.q.out ql/src/test/results/clientpositive/perf/query51.q.out index 55e08d9..9695d99 100644 --- ql/src/test/results/clientpositive/perf/query51.q.out +++ ql/src/test/results/clientpositive/perf/query51.q.out @@ -87,109 +87,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_53] - Limit [LIM_52] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_51] (rows=116159124 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_50] - Select Operator [SEL_46] (rows=116159124 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_58] (rows=116159124 width=88) - predicate:(max_window_0 > max_window_1) - PTF Operator [PTF_45] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"CASE WHEN (_col4 is not null) THEN (_col4) ELSE (_col1) END ASC NULLS FIRST","partition by:":"CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END"}] - Select Operator [SEL_44] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:CASE WHEN (_col3 is not null) THEN (_col3) ELSE (_col0) END - Merge Join Operator [MERGEJOIN_67] (rows=348477374 width=88) - Conds:RS_40._col0, _col1=RS_41._col0, _col1(Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Select Operator [SEL_37] (rows=79201469 width=135) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_36] (rows=79201469 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_32] (rows=79201469 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=158402938 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Select Operator [SEL_29] (rows=158402938 width=135) - Output:["_col1","_col4","_col2"] - Merge Join Operator [MERGEJOIN_66] (rows=158402938 width=135) - Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_62] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_23] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_61] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_20] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_sales_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Select Operator [SEL_17] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"] - PTF Operator [PTF_16] (rows=316797606 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}] - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col1, _col4 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col1","_col4","_col2"] - Merge Join Operator [MERGEJOIN_65] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_59] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_60] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query52.q.out ql/src/test/results/clientpositive/perf/query52.q.out index c61ed30..54dd921 100644 --- ql/src/test/results/clientpositive/perf/query52.q.out +++ ql/src/test/results/clientpositive/perf/query52.q.out @@ -5,66 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col1","_col2","_col3"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query54.q.out ql/src/test/results/clientpositive/perf/query54.q.out index 0b59b1b..5794748 100644 --- ql/src/test/results/clientpositive/perf/query54.q.out +++ ql/src/test/results/clientpositive/perf/query54.q.out @@ -5,175 +5,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 12 <- Union 13 (CONTAINS) -Map 18 <- Union 13 (CONTAINS) -Reducer 10 <- Reducer 17 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 19 (SIMPLE_EDGE), Union 13 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 13 (CONTAINS) +Vertex 14 <- Union 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 13 (CONTAINS) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_69] - Limit [LIM_68] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_67] (rows=174238687 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_66] - Select Operator [SEL_65] (rows=174238687 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_64] (rows=174238687 width=88) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Group By Operator [GBY_62] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col0 - Select Operator [SEL_60] (rows=348477374 width=88) - Output:["_col0"] - Group By Operator [GBY_59] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col10 - Select Operator [SEL_56] (rows=696954748 width=88) - Output:["_col10","_col2"] - Merge Join Operator [MERGEJOIN_117] (rows=696954748 width=88) - Conds:RS_53._col1=RS_54._col5(Inner),Output:["_col2","_col10"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_116] (rows=316240138 width=135) - Conds:RS_46._col0=RS_47._col1(Inner),Output:["_col5"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col1 - Select Operator [SEL_42] (rows=287491029 width=135) - Output:["_col0","_col1"] - Group By Operator [GBY_41] (rows=287491029 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Group By Operator [GBY_39] (rows=574982058 width=135) - Output:["_col0","_col1"],keys:_col10, _col9 - Select Operator [SEL_38] (rows=574982058 width=135) - Output:["_col10","_col9"] - Merge Join Operator [MERGEJOIN_115] (rows=574982058 width=135) - Conds:RS_35._col1=RS_36._col0(Inner),Output:["_col9","_col10"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_110] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_26] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_114] (rows=522710951 width=135) - Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col1"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=115500 width=1436) - Output:["_col0"] - Filter Operator [FIL_109] (rows=115500 width=1436) - predicate:((i_category = 'Jewelry') and (i_class = 'football') and i_item_sk is not null) - TableScan [TS_23] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_113] (rows=475191764 width=135) - Conds:Union 13._col0=RS_30._col0(Inner),Output:["_col1","_col2"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_108] (rows=18262 width=1119) - predicate:((d_moy = 3) and (d_year = 2000) and d_date_sk is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Union 13 [SIMPLE_EDGE] - <-Map 12 [CONTAINS] - Reduce Output Operator [RS_29] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_106] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_12] (rows=287989836 width=135) - Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] - <-Map 18 [CONTAINS] - Reduce Output Operator [RS_29] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_107] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_15] (rows=144002668 width=135) - Output:["ws_sold_date_sk","ws_item_sk","ws_bill_customer_sk"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_112] (rows=44000000 width=1014) - Conds:RS_43._col1, _col2=RS_44._col0, _col1(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0, _col1 - Select Operator [SEL_11] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_105] (rows=1704 width=1910) - predicate:(s_county is not null and s_state is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_county","s_state"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1, _col2 - Select Operator [SEL_8] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_104] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null and ca_state is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_111] (rows=633595212 width=88) - Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_102] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_103] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1203 AND 1205 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query55.q.out ql/src/test/results/clientpositive/perf/query55.q.out index fb3743d..dc78857 100644 --- ql/src/test/results/clientpositive/perf/query55.q.out +++ ql/src/test/results/clientpositive/perf/query55.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) - predicate:((i_manager_id = 36) and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query56.q.out ql/src/test/results/clientpositive/perf/query56.q.out index 4fa28c2..0035d9f 100644 --- ql/src/test/results/clientpositive/perf/query56.q.out +++ ql/src/test/results/clientpositive/perf/query56.q.out @@ -133,289 +133,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query58.q.out ql/src/test/results/clientpositive/perf/query58.q.out index f7e6a08..e6820ca 100644 --- ql/src/test/results/clientpositive/perf/query58.q.out +++ ql/src/test/results/clientpositive/perf/query58.q.out @@ -95,270 +95,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Map 23 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 26 <- Map 32 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Map 33 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE) -Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 29 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 18 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 28 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_113] - Limit [LIM_112] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_111] (rows=14606 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_110] - Select Operator [SEL_109] (rows=14606 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_108] (rows=14606 width=135) - predicate:(_col1 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col3 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col5 BETWEEN (0.9 * _col1) AND (1.1 * _col1) and _col5 BETWEEN (0.9 * _col3) AND (1.1 * _col3)) - Merge Join Operator [MERGEJOIN_209] (rows=95833780 width=135) - Conds:RS_104._col0=RS_105._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Group By Operator [GBY_102] (rows=87121617 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Group By Operator [GBY_100] (rows=174243235 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_99] (rows=174243235 width=135) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_207] (rows=174243235 width=135) - Conds:RS_96._col4=RS_97._col0(Inner),Output:["_col5","_col7"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_195] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_87] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_206] (rows=158402938 width=135) - Conds:RS_93._col0=RS_94._col0(Inner),Output:["_col4","_col5"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_86] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_194] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_84] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_205] (rows=88388 width=1119) - Conds:RS_90._col1=RS_91._col0(Inner),Output:["_col0"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col1 - Select Operator [SEL_73] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_71] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_204] (rows=80353 width=1119) - Conds:RS_80._col1=RS_81._col1(Inner),Output:["_col0"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col1 - Select Operator [SEL_76] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_192] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_74] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col1 - Select Operator [SEL_79] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_193] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_77] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Filter Operator [FIL_69] (rows=4732408 width=88) - predicate:(_col1 BETWEEN (0.9 * _col3) AND (1.1 * _col3) and _col3 BETWEEN (0.9 * _col1) AND (1.1 * _col1)) - Merge Join Operator [MERGEJOIN_208] (rows=383325119 width=88) - Conds:RS_66._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col0 - Group By Operator [GBY_64] (rows=174233858 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0 - Group By Operator [GBY_62] (rows=348467716 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_61] (rows=348467716 width=135) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_203] (rows=348467716 width=135) - Conds:RS_58._col4=RS_59._col0(Inner),Output:["_col5","_col7"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_190] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_49] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_202] (rows=316788826 width=135) - Conds:RS_55._col0=RS_56._col0(Inner),Output:["_col4","_col5"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_189] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_46] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_201] (rows=88388 width=1119) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_186] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_33] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_200] (rows=80353 width=1119) - Conds:RS_42._col1=RS_43._col1(Inner),Output:["_col0"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_187] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_36] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_41] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_188] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_39] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Group By Operator [GBY_29] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col5)"],keys:_col7 - Select Operator [SEL_28] (rows=696954748 width=88) - Output:["_col7","_col5"] - Merge Join Operator [MERGEJOIN_199] (rows=696954748 width=88) - Conds:RS_25._col4=RS_26._col0(Inner),Output:["_col5","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_185] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_16] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_198] (rows=633595212 width=88) - Conds:RS_22._col0=RS_23._col0(Inner),Output:["_col4","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_184] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_13] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_197] (rows=88388 width=1119) - Conds:RS_19._col1=RS_20._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_181] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_date is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_196] (rows=80353 width=1119) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col1"] - Filter Operator [FIL_183] (rows=36524 width=1119) - predicate:((d_date = '1998-08-04') and d_week_seq is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_182] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query6.q.out ql/src/test/results/clientpositive/perf/query6.q.out index 094459e..79b6220 100644 --- ql/src/test/results/clientpositive/perf/query6.q.out +++ ql/src/test/results/clientpositive/perf/query6.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join MERGEJOIN[196][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[193][tables = [$hdt$_5, $hdt$_6]] in Stage 'Reducer 21' is a cross product -Warning: Shuffle Join MERGEJOIN[194][tables = [$hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Reducer 22' is a cross product -Warning: Shuffle Join MERGEJOIN[195][tables = [$hdt$_5, $hdt$_6, $hdt$_7, $hdt$_8, $hdt$_9, $hdt$_10, $hdt$_11]] in Stage 'Reducer 23' is a cross product +Warning: Shuffle Join MERGEJOIN[196][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 2' is a cross product +Warning: Shuffle Join MERGEJOIN[193][tables = [$hdt$_5, $hdt$_6]] in Stage 'Vertex 21' is a cross product +Warning: Shuffle Join MERGEJOIN[194][tables = [$hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Vertex 22' is a cross product +Warning: Shuffle Join MERGEJOIN[195][tables = [$hdt$_5, $hdt$_6, $hdt$_7, $hdt$_8, $hdt$_9, $hdt$_10, $hdt$_11]] in Stage 'Vertex 23' is a cross product PREHOOK: query: explain select a.ca_state state, count(*) cnt from customer_address a ,customer c @@ -53,280 +53,31 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) -Reducer 12 <- Map 11 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 18 <- Map 17 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 10 (CUSTOM_SIMPLE_EDGE) -Reducer 21 <- Map 20 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (CUSTOM_SIMPLE_EDGE), Reducer 29 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Map 30 (CUSTOM_SIMPLE_EDGE), Map 31 (CUSTOM_SIMPLE_EDGE), Map 32 (CUSTOM_SIMPLE_EDGE), Map 33 (CUSTOM_SIMPLE_EDGE), Reducer 22 (CUSTOM_SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (CUSTOM_SIMPLE_EDGE) -Reducer 29 <- Map 28 (SIMPLE_EDGE) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 36 (SIMPLE_EDGE) -Reducer 4 <- Reducer 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 35 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 10 (PARTITION_EDGE) +Vertex 21 <- Vertex 20 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 29 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE), Vertex 31 (PARTITION_EDGE), Vertex 32 (PARTITION_EDGE), Vertex 33 (PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_129] - Limit [LIM_128] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_127] (rows=187554724582209678 width=1) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_126] - Filter Operator [FIL_124] (rows=187554724582209678 width=1) - predicate:(_col1 >= 10) - Group By Operator [GBY_123] (rows=562664173746629034 width=1) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Group By Operator [GBY_121] (rows=1125328347493258069 width=1) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col1 - Select Operator [SEL_120] (rows=1125328347493258069 width=1) - Output:["_col1"] - Filter Operator [FIL_119] (rows=1125328347493258069 width=1) - predicate:(_col10 > (1.2 * CASE WHEN (_col15 is null) THEN (null) ELSE (_col14) END)) - Select Operator [SEL_118] (rows=3375985042479774208 width=1) - Output:["_col1","_col10","_col14","_col15"] - Merge Join Operator [MERGEJOIN_203] (rows=3375985042479774208 width=1) - Conds:RS_115._col2=RS_116._col0(Inner),Output:["_col8","_col10","_col11","_col16"] - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_200] (rows=88000001 width=860) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col3"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_98] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_customer_sk is not null) - TableScan [TS_96] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_192] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_99] (rows=40000000 width=1014) - default@customer_address,a,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_202] (rows=3069077244824791552 width=1) - Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col2","_col8","_col10","_col11"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_199] (rows=2790070162094850048 width=1) - Conds:RS_92._col2=RS_93._col2(Left Outer),Output:["_col0","_col1","_col3","_col4"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col2 - Select Operator [SEL_33] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_186] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_31] (rows=462000 width=1436) - default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_category"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col2 - Select Operator [SEL_91] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_90] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Group By Operator [GBY_88] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],aggregations:["avg(_col0)"],keys:_col2 - Select Operator [SEL_87] (rows=5072854730221289472 width=1) - Output:["_col2","_col0"] - Merge Join Operator [MERGEJOIN_198] (rows=5072854730221289472 width=1) - Conds:RS_84._col1=RS_85._col0(Inner),Output:["_col0","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_84] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=462000 width=1436) - Output:["_col0","_col1"] - TableScan [TS_34] (rows=462000 width=1436) - default@item,j,Tbl:COMPLETE,Col:NONE,Output:["i_current_price","i_category"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_85] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0 - Group By Operator [GBY_80] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col3 - Merge Join Operator [MERGEJOIN_195] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),(Inner),Output:["_col3"] - <-Map 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_74] - Select Operator [SEL_60] (rows=462000 width=1436) - Output:["_col0"] - TableScan [TS_59] (rows=462000 width=1436) - default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_category"] - <-Map 31 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_75] - Select Operator [SEL_62] (rows=40000000 width=4) - TableScan [TS_61] (rows=40000000 width=1014) - default@customer_address,a,Tbl:COMPLETE,Col:COMPLETE - <-Map 32 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_76] - Select Operator [SEL_64] (rows=80000000 width=4) - TableScan [TS_63] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:COMPLETE - <-Map 33 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_77] - Select Operator [SEL_66] (rows=575995635 width=4) - TableScan [TS_65] (rows=575995635 width=88) - default@store_sales,s,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 22 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_73] - Merge Join Operator [MERGEJOIN_194] (rows=667010419 width=1133) - Conds:(Left Outer) - <-Reducer 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_70] - Merge Join Operator [MERGEJOIN_193] (rows=73049 width=13) - Conds:(Inner) - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_67] - Select Operator [SEL_37] (rows=73049 width=4) - TableScan [TS_36] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_68] - Select Operator [SEL_51] (rows=1 width=8) - Filter Operator [FIL_50] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 26 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_47] - Group By Operator [GBY_46] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_44] (rows=9131 width=1119) - Group By Operator [GBY_43] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Group By Operator [GBY_41] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_40] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_188] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2)) - TableScan [TS_38] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] - <-Reducer 29 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_58] (rows=9131 width=1119) - Group By Operator [GBY_57] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Group By Operator [GBY_55] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_54] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_189] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2)) - TableScan [TS_52] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_201] (rows=633595212 width=97) - Conds:RS_109._col0=RS_110._col0(Inner),Output:["_col1","_col2"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_197] (rows=80353 width=1119) - Conds:RS_27._col1=RS_28._col0(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_184] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_month_seq is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_22] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_185] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2) and d_month_seq is not null) - TableScan [TS_20] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_196] (rows=575995635 width=97) - Conds:(Inner),Output:["_col0","_col1","_col2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_106] - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_182] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,s,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Reducer 10 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_107] - Select Operator [SEL_16] (rows=1 width=8) - Filter Operator [FIL_15] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_13] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 9 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_12] - Group By Operator [GBY_11] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_9] (rows=9131 width=1119) - Group By Operator [GBY_8] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=18262 width=1119) - Output:["_col0"],keys:d_month_seq - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["d_month_seq"] - Filter Operator [FIL_183] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 2)) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query60.q.out ql/src/test/results/clientpositive/perf/query60.q.out index ad9d08e..f050ef6 100644 --- ql/src/test/results/clientpositive/perf/query60.q.out +++ ql/src/test/results/clientpositive/perf/query60.q.out @@ -153,289 +153,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) -Reducer 23 <- Map 25 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 3 <- Reducer 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Map 30 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 34 <- Map 36 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 5 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_122] - Limit [LIM_121] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_120] (rows=335408073 width=108) - Output:["_col0","_col1"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_119] - Group By Operator [GBY_117] (rows=335408073 width=108) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 18 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_177] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_165] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - TableScan [TS_40] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_179] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_168] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_53] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_178] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_166] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_addr_sk is not null and cs_item_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_167] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_50] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_111] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_185] (rows=191667562 width=135) - Conds:RS_105._col0=RS_106._col3(Inner),Output:["_col1","_col8"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_180] (rows=508200 width=1436) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0","_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_169] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_76] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Group By Operator [GBY_84] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_81] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_170] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col3 - Select Operator [SEL_101] (rows=174243235 width=135) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_182] (rows=174243235 width=135) - Conds:RS_98._col2=RS_99._col0(Inner),Output:["_col1","_col3"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_173] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_92] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=158402938 width=135) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null and ws_item_sk is not null) - TableScan [TS_86] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_89] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=766650239 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=766650239 width=88) - Conds:RS_29._col0=RS_30._col3(Inner),Output:["_col1","_col8"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Select Operator [SEL_25] (rows=696954748 width=88) - Output:["_col3","_col5"] - Merge Join Operator [MERGEJOIN_176] (rows=696954748 width=88) - Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_163] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_175] (rows=633595212 width=88) - Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_161] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null and ss_item_sk is not null) - TableScan [TS_10] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_174] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_159] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_160] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query64.q.out ql/src/test/results/clientpositive/perf/query64.q.out index 468c1bc..115e5e6 100644 --- ql/src/test/results/clientpositive/perf/query64.q.out +++ ql/src/test/results/clientpositive/perf/query64.q.out @@ -5,657 +5,52 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 12 <- Map 40 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Map 41 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 17 <- Map 23 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 24 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Map 34 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 35 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE), Map 30 (SIMPLE_EDGE) -Reducer 27 <- Map 31 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Map 32 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 33 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 37 <- Map 36 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 44 <- Reducer 43 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 45 <- Reducer 44 (SIMPLE_EDGE) -Reducer 48 <- Map 47 (SIMPLE_EDGE), Map 53 (SIMPLE_EDGE) -Reducer 49 <- Reducer 48 (SIMPLE_EDGE), Reducer 60 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 45 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (SIMPLE_EDGE), Reducer 77 (SIMPLE_EDGE) -Reducer 51 <- Map 79 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE) -Reducer 52 <- Map 80 (SIMPLE_EDGE), Reducer 51 (SIMPLE_EDGE) -Reducer 55 <- Map 54 (SIMPLE_EDGE), Map 61 (SIMPLE_EDGE) -Reducer 56 <- Map 62 (SIMPLE_EDGE), Reducer 55 (SIMPLE_EDGE) -Reducer 57 <- Map 63 (SIMPLE_EDGE), Reducer 56 (SIMPLE_EDGE) -Reducer 58 <- Reducer 57 (SIMPLE_EDGE), Reducer 68 (SIMPLE_EDGE) -Reducer 59 <- Map 73 (SIMPLE_EDGE), Reducer 58 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 60 <- Map 74 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE) -Reducer 65 <- Map 64 (SIMPLE_EDGE), Map 69 (SIMPLE_EDGE) -Reducer 66 <- Map 70 (SIMPLE_EDGE), Reducer 65 (SIMPLE_EDGE) -Reducer 67 <- Map 71 (SIMPLE_EDGE), Reducer 66 (SIMPLE_EDGE) -Reducer 68 <- Map 72 (SIMPLE_EDGE), Reducer 67 (SIMPLE_EDGE) -Reducer 76 <- Map 75 (SIMPLE_EDGE), Map 78 (SIMPLE_EDGE) -Reducer 77 <- Reducer 76 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 52 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (SORT_PARTITION_EDGE) +Vertex 48 <- Vertex 47 (SORT_PARTITION_EDGE), Vertex 53 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (SORT_PARTITION_EDGE), Vertex 60 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 45 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (SORT_PARTITION_EDGE), Vertex 77 (SORT_PARTITION_EDGE) +Vertex 51 <- Vertex 50 (SORT_PARTITION_EDGE), Vertex 79 (SORT_PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 80 (SORT_PARTITION_EDGE) +Vertex 55 <- Vertex 54 (SORT_PARTITION_EDGE), Vertex 61 (SORT_PARTITION_EDGE) +Vertex 56 <- Vertex 55 (SORT_PARTITION_EDGE), Vertex 62 (SORT_PARTITION_EDGE) +Vertex 57 <- Vertex 56 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 58 <- Vertex 57 (SORT_PARTITION_EDGE), Vertex 68 (SORT_PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE), Vertex 73 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 60 <- Vertex 59 (SORT_PARTITION_EDGE), Vertex 74 (SORT_PARTITION_EDGE) +Vertex 65 <- Vertex 64 (SORT_PARTITION_EDGE), Vertex 69 (SORT_PARTITION_EDGE) +Vertex 66 <- Vertex 65 (SORT_PARTITION_EDGE), Vertex 70 (SORT_PARTITION_EDGE) +Vertex 67 <- Vertex 66 (SORT_PARTITION_EDGE), Vertex 71 (SORT_PARTITION_EDGE) +Vertex 68 <- Vertex 67 (SORT_PARTITION_EDGE), Vertex 72 (SORT_PARTITION_EDGE) +Vertex 76 <- Vertex 75 (SORT_PARTITION_EDGE), Vertex 78 (SORT_PARTITION_EDGE) +Vertex 77 <- Vertex 76 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_267] - Select Operator [SEL_266] (rows=331415616 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_265] - Select Operator [SEL_264] (rows=331415616 width=88) - Output:["_col0","_col1","_col10","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col2","_col20","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_263] (rows=331415616 width=88) - predicate:(_col30 <= _col13) - Merge Join Operator [MERGEJOIN_658] (rows=994246850 width=88) - Conds:RS_260._col2, _col1, _col3=RS_261._col2, _col1, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col30","_col31","_col32","_col33"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_260] - PartitionCols:_col2, _col1, _col3 - Select Operator [SEL_128] (rows=903860754 width=88) - Output:["_col0","_col1","_col10","_col11","_col13","_col14","_col15","_col16","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Group By Operator [GBY_127] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Group By Operator [GBY_125] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col44, _col50, _col45, _col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col53 - Select Operator [SEL_124] (rows=1807721509 width=88) - Output:["_col44","_col50","_col45","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col53","_col26","_col27","_col28"] - Merge Join Operator [MERGEJOIN_656] (rows=1807721509 width=88) - Conds:RS_121._col0=RS_122._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col18 - Select Operator [SEL_117] (rows=1643383155 width=88) - Output:["_col1","_col18","_col2","_col23","_col24","_col25","_col3","_col37","_col39","_col4","_col41","_col42","_col47","_col50","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_638] (rows=1643383155 width=88) - Conds:RS_114._col21=RS_115._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48","_col50","_col51","_col52","_col53"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_602] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_99] (rows=40000000 width=1014) - default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_114] - PartitionCols:_col21 - Merge Join Operator [MERGEJOIN_637] (rows=1493984654 width=88) - Conds:RS_111._col9=RS_112._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_98] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_601] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_96] (rows=40000000 width=1014) - default@customer_address,ad1,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_636] (rows=1358167838 width=88) - Conds:RS_108._col5=RS_109._col0(Inner),Output:["_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_635] (rows=1234698008 width=88) - Conds:RS_105._col0=RS_106._col17(Inner),Output:["_col5","_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col17 - Select Operator [SEL_79] (rows=1122452711 width=88) - Output:["_col10","_col11","_col12","_col17","_col18","_col2","_col24","_col26","_col28","_col29","_col34","_col37","_col5","_col6"] - Merge Join Operator [MERGEJOIN_633] (rows=1122452711 width=88) - Conds:RS_76._col7=RS_77._col0(Inner),Output:["_col1","_col4","_col5","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_598] (rows=2300 width=1179) - predicate:p_promo_sk is not null - TableScan [TS_70] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_632] (rows=1020411534 width=88) - Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_69] (rows=2851 width=1436) - Output:["_col0","_col3"] - Filter Operator [FIL_597] (rows=2851 width=1436) - predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) - TableScan [TS_67] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col1 - Select Operator [SEL_66] (rows=927646829 width=88) - Output:["_col1","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col4","_col5","_col7","_col9"] - Filter Operator [FIL_65] (rows=927646829 width=88) - predicate:(_col30 <> _col32) - Select Operator [SEL_64] (rows=927646829 width=88) - Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col30","_col32"] - Merge Join Operator [MERGEJOIN_631] (rows=927646829 width=88) - Conds:RS_61._col0=RS_62._col4(Inner),Output:["_col2","_col3","_col7","_col9","_col11","_col13","_col15","_col18","_col19","_col21","_col23","_col24","_col25","_col31","_col32"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_626] (rows=106480005 width=860) - Conds:RS_58._col1=RS_59._col0(Inner),Output:["_col0","_col2","_col3","_col7","_col9","_col11"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_23] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_591] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_21] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_625] (rows=96800003 width=860) - Conds:RS_55._col4=RS_56._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_590] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_18] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_624] (rows=88000001 width=860) - Conds:RS_52._col5=RS_53._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col5 - Select Operator [SEL_14] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_588] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) - TableScan [TS_12] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_589] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_15] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col4 - Select Operator [SEL_51] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col13","_col19","_col20","_col3","_col4","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_630] (rows=843315281 width=88) - Conds:RS_48._col3=RS_49._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18","_col20"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_596] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_36] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_629] (rows=766650239 width=88) - Conds:RS_45._col6=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_595] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) - TableScan [TS_33] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_628] (rows=696954748 width=88) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_594] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - TableScan [TS_30] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_627] (rows=633595212 width=88) - Conds:RS_39._col1, _col8=RS_40._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1, _col8 - Select Operator [SEL_26] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_592] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_customer_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) - TableScan [TS_24] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Map 30 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0, _col1 - Select Operator [SEL_29] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_593] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_27] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_623] (rows=7920 width=107) - Conds:RS_102._col1=RS_103._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_587] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_9] (rows=20 width=12) - default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_586] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_6] (rows=7200 width=107) - default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_95] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_94] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_93] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col0 - Group By Operator [GBY_91] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_89] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_634] (rows=316788826 width=135) - Conds:RS_86._col0, _col1=RS_87._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0, _col1 - Select Operator [SEL_82] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_599] (rows=287989836 width=135) - predicate:(cs_order_number is not null and cs_item_sk is not null) - TableScan [TS_80] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_600] (rows=28798881 width=106) - predicate:(cr_order_number is not null and cr_item_sk is not null) - TableScan [TS_83] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_121] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_622] (rows=7920 width=107) - Conds:RS_118._col1=RS_119._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_584] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_0] (rows=7200 width=107) - default@household_demographics,hd1,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_585] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_3] (rows=20 width=12) - default@income_band,ib1,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 45 [SIMPLE_EDGE] - SHUFFLE [RS_261] - PartitionCols:_col2, _col1, _col3 - Select Operator [SEL_258] (rows=903860754 width=88) - Output:["_col1","_col13","_col14","_col15","_col16","_col2","_col3"] - Group By Operator [GBY_257] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_256] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Group By Operator [GBY_255] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col44, _col50, _col45, _col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col53 - Select Operator [SEL_254] (rows=1807721509 width=88) - Output:["_col44","_col50","_col45","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col53","_col26","_col27","_col28"] - Merge Join Operator [MERGEJOIN_657] (rows=1807721509 width=88) - Conds:RS_251._col0=RS_252._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_251] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_639] (rows=7920 width=107) - Conds:RS_248._col1=RS_249._col0(Inner),Output:["_col0"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_248] - PartitionCols:_col1 - Select Operator [SEL_132] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_603] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_130] (rows=7200 width=107) - default@household_demographics,hd1,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_249] - PartitionCols:_col0 - Select Operator [SEL_135] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_604] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_133] (rows=20 width=12) - default@income_band,ib1,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_252] - PartitionCols:_col18 - Select Operator [SEL_247] (rows=1643383155 width=88) - Output:["_col1","_col18","_col2","_col23","_col24","_col25","_col3","_col37","_col39","_col4","_col41","_col42","_col47","_col50","_col6","_col7","_col8","_col9"] - Merge Join Operator [MERGEJOIN_655] (rows=1643383155 width=88) - Conds:RS_244._col21=RS_245._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48","_col50","_col51","_col52","_col53"] - <-Map 80 [SIMPLE_EDGE] - SHUFFLE [RS_245] - PartitionCols:_col0 - Select Operator [SEL_231] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_621] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_229] (rows=40000000 width=1014) - default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 51 [SIMPLE_EDGE] - SHUFFLE [RS_244] - PartitionCols:_col21 - Merge Join Operator [MERGEJOIN_654] (rows=1493984654 width=88) - Conds:RS_241._col9=RS_242._col0(Inner),Output:["_col8","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40","_col45","_col46","_col47","_col48"] - <-Map 79 [SIMPLE_EDGE] - SHUFFLE [RS_242] - PartitionCols:_col0 - Select Operator [SEL_228] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_620] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_226] (rows=40000000 width=1014) - default@customer_address,ad1,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 50 [SIMPLE_EDGE] - SHUFFLE [RS_241] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_653] (rows=1358167838 width=88) - Conds:RS_238._col5=RS_239._col0(Inner),Output:["_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 49 [SIMPLE_EDGE] - SHUFFLE [RS_238] - PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_652] (rows=1234698008 width=88) - Conds:RS_235._col0=RS_236._col17(Inner),Output:["_col5","_col8","_col9","_col13","_col14","_col15","_col21","_col27","_col29","_col31","_col32","_col37","_col40"] - <-Reducer 48 [SIMPLE_EDGE] - SHUFFLE [RS_235] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_640] (rows=7920 width=107) - Conds:RS_232._col1=RS_233._col0(Inner),Output:["_col0"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_232] - PartitionCols:_col1 - Select Operator [SEL_138] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_605] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_136] (rows=7200 width=107) - default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 53 [SIMPLE_EDGE] - SHUFFLE [RS_233] - PartitionCols:_col0 - Select Operator [SEL_141] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_606] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_139] (rows=20 width=12) - default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 60 [SIMPLE_EDGE] - SHUFFLE [RS_236] - PartitionCols:_col17 - Select Operator [SEL_209] (rows=1122452711 width=88) - Output:["_col10","_col11","_col12","_col17","_col18","_col2","_col24","_col26","_col28","_col29","_col34","_col37","_col5","_col6"] - Merge Join Operator [MERGEJOIN_650] (rows=1122452711 width=88) - Conds:RS_206._col7=RS_207._col0(Inner),Output:["_col1","_col4","_col5","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 74 [SIMPLE_EDGE] - SHUFFLE [RS_207] - PartitionCols:_col0 - Select Operator [SEL_202] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_617] (rows=2300 width=1179) - predicate:p_promo_sk is not null - TableScan [TS_200] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 59 [SIMPLE_EDGE] - SHUFFLE [RS_206] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_649] (rows=1020411534 width=88) - Conds:RS_203._col1=RS_204._col0(Inner),Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col33","_col36"] - <-Map 73 [SIMPLE_EDGE] - SHUFFLE [RS_204] - PartitionCols:_col0 - Select Operator [SEL_199] (rows=2851 width=1436) - Output:["_col0","_col3"] - Filter Operator [FIL_616] (rows=2851 width=1436) - predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) - TableScan [TS_197] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] - <-Reducer 58 [SIMPLE_EDGE] - SHUFFLE [RS_203] - PartitionCols:_col1 - Select Operator [SEL_196] (rows=927646829 width=88) - Output:["_col1","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col4","_col5","_col7","_col9"] - Filter Operator [FIL_195] (rows=927646829 width=88) - predicate:(_col30 <> _col32) - Select Operator [SEL_194] (rows=927646829 width=88) - Output:["_col1","_col4","_col5","_col7","_col9","_col10","_col11","_col16","_col17","_col23","_col25","_col27","_col28","_col30","_col32"] - Merge Join Operator [MERGEJOIN_648] (rows=927646829 width=88) - Conds:RS_191._col0=RS_192._col4(Inner),Output:["_col2","_col3","_col7","_col9","_col11","_col13","_col15","_col18","_col19","_col21","_col23","_col24","_col25","_col31","_col32"] - <-Reducer 57 [SIMPLE_EDGE] - SHUFFLE [RS_191] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_643] (rows=106480005 width=860) - Conds:RS_188._col1=RS_189._col0(Inner),Output:["_col0","_col2","_col3","_col7","_col9","_col11"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_189] - PartitionCols:_col0 - Select Operator [SEL_153] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_610] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_151] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 56 [SIMPLE_EDGE] - SHUFFLE [RS_188] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_642] (rows=96800003 width=860) - Conds:RS_185._col4=RS_186._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 62 [SIMPLE_EDGE] - SHUFFLE [RS_186] - PartitionCols:_col0 - Select Operator [SEL_150] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_609] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_148] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 55 [SIMPLE_EDGE] - SHUFFLE [RS_185] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_641] (rows=88000001 width=860) - Conds:RS_182._col5=RS_183._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 54 [SIMPLE_EDGE] - SHUFFLE [RS_182] - PartitionCols:_col5 - Select Operator [SEL_144] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_607] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) - TableScan [TS_142] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] - <-Map 61 [SIMPLE_EDGE] - SHUFFLE [RS_183] - PartitionCols:_col0 - Select Operator [SEL_147] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_608] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_145] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 68 [SIMPLE_EDGE] - SHUFFLE [RS_192] - PartitionCols:_col4 - Select Operator [SEL_181] (rows=843315281 width=88) - Output:["_col1","_col11","_col12","_col13","_col19","_col20","_col3","_col4","_col6","_col7","_col9"] - Merge Join Operator [MERGEJOIN_647] (rows=843315281 width=88) - Conds:RS_178._col3=RS_179._col0(Inner),Output:["_col1","_col2","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18","_col20"] - <-Map 72 [SIMPLE_EDGE] - SHUFFLE [RS_179] - PartitionCols:_col0 - Select Operator [SEL_168] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_615] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_166] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 67 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_646] (rows=766650239 width=88) - Conds:RS_175._col6=RS_176._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col17","_col18"] - <-Map 71 [SIMPLE_EDGE] - SHUFFLE [RS_176] - PartitionCols:_col0 - Select Operator [SEL_165] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_614] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) - TableScan [TS_163] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 66 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col6 - Merge Join Operator [MERGEJOIN_645] (rows=696954748 width=88) - Conds:RS_172._col0=RS_173._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 70 [SIMPLE_EDGE] - SHUFFLE [RS_173] - PartitionCols:_col0 - Select Operator [SEL_162] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_613] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_160] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 65 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_644] (rows=633595212 width=88) - Conds:RS_169._col1, _col8=RS_170._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col1, _col8 - Select Operator [SEL_156] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_611] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_customer_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) - TableScan [TS_154] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Map 69 [SIMPLE_EDGE] - SHUFFLE [RS_170] - PartitionCols:_col0, _col1 - Select Operator [SEL_159] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_612] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_157] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 77 [SIMPLE_EDGE] - SHUFFLE [RS_239] - PartitionCols:_col0 - Select Operator [SEL_225] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_224] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_223] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 76 [SIMPLE_EDGE] - SHUFFLE [RS_222] - PartitionCols:_col0 - Group By Operator [GBY_221] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_219] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_651] (rows=316788826 width=135) - Conds:RS_216._col0, _col1=RS_217._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 75 [SIMPLE_EDGE] - SHUFFLE [RS_216] - PartitionCols:_col0, _col1 - Select Operator [SEL_212] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_618] (rows=287989836 width=135) - predicate:(cs_order_number is not null and cs_item_sk is not null) - TableScan [TS_210] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] - <-Map 78 [SIMPLE_EDGE] - SHUFFLE [RS_217] - PartitionCols:_col0, _col1 - Select Operator [SEL_215] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_619] (rows=28798881 width=106) - predicate:(cr_order_number is not null and cr_item_sk is not null) - TableScan [TS_213] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query65.q.out ql/src/test/results/clientpositive/perf/query65.q.out index 060cddf..af6d33f 100644 --- ql/src/test/results/clientpositive/perf/query65.q.out +++ ql/src/test/results/clientpositive/perf/query65.q.out @@ -77,125 +77,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_52] - Limit [LIM_51] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_50] (rows=255550079 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_49] - Select Operator [SEL_48] (rows=255550079 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_47] (rows=255550079 width=88) - predicate:(_col11 <= (0.1 * _col8)) - Select Operator [SEL_46] (rows=766650239 width=88) - Output:["_col1","_col3","_col4","_col5","_col6","_col8","_col11"] - Merge Join Operator [MERGEJOIN_82] (rows=766650239 width=88) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col4","_col6","_col8","_col9","_col10","_col11"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_78] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_36] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc","i_current_price","i_wholesale_cost","i_brand"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_81] (rows=696954748 width=88) - Conds:RS_39._col0=RS_40._col0(Inner),RS_39._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col4","_col6"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_33] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_32] (rows=158398803 width=88) - Output:["_col0","_col1"] - Group By Operator [GBY_31] (rows=158398803 width=88) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Select Operator [SEL_27] (rows=316797606 width=88) - Output:["_col1","_col2"] - Group By Operator [GBY_26] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_23] (rows=633595212 width=88) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_80] (rows=633595212 width=88) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_75] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_14] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1","_col3"] - Merge Join Operator [MERGEJOIN_79] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_73] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_74] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query66.q.out ql/src/test/results/clientpositive/perf/query66.q.out index 5cf60f6..6fd32bb 100644 --- ql/src/test/results/clientpositive/perf/query66.q.out +++ ql/src/test/results/clientpositive/perf/query66.q.out @@ -439,190 +439,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Map 22 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 23 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 7 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 7 (CONTAINS), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 9 - File Output Operator [FS_76] - Limit [LIM_75] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_74] (rows=158120068 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_73] - Select Operator [SEL_72] (rows=158120068 width=135) - Output:["_col0","_col1","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col2","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col3","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col4","_col40","_col41","_col42","_col43","_col5","_col8","_col9"] - Group By Operator [GBY_71] (rows=158120068 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)","sum(VALUE._col24)","sum(VALUE._col25)","sum(VALUE._col26)","sum(VALUE._col27)","sum(VALUE._col28)","sum(VALUE._col29)","sum(VALUE._col30)","sum(VALUE._col31)","sum(VALUE._col32)","sum(VALUE._col33)","sum(VALUE._col34)","sum(VALUE._col35)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] - Group By Operator [GBY_64] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_62] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_60] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Merge Join Operator [MERGEJOIN_122] (rows=421645953 width=135) - Conds:RS_57._col3=RS_58._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_114] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_45] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_121] (rows=383314495 width=135) - Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_113] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_42] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_120] (rows=348467716 width=135) - Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_112] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_39] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_119] (rows=316788826 width=135) - Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_110] (rows=287989836 width=135) - predicate:(cs_warehouse_sk is not null and cs_sold_date_sk is not null and cs_sold_time_sk is not null and cs_ship_mode_sk is not null) - TableScan [TS_33] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_ship_mode_sk","cs_warehouse_sk","cs_quantity","cs_ext_sales_price","cs_net_paid_inc_ship_tax"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_111] (rows=9600 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_36] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] - <-Reducer 6 [CONTAINS] - Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] - Group By Operator [GBY_31] (rows=105417161 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_29] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_27] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_109] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_12] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_117] (rows=191667562 width=135) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_108] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_9] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_116] (rows=174243235 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_107] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_115] (rows=158402938 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_105] (rows=144002668 width=135) - predicate:(ws_warehouse_sk is not null and ws_sold_date_sk is not null and ws_sold_time_sk is not null and ws_ship_mode_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_ship_mode_sk","ws_warehouse_sk","ws_quantity","ws_sales_price","ws_net_paid_inc_tax"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_106] (rows=9600 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] + Vertex 9 diff --git ql/src/test/results/clientpositive/perf/query67.q.out ql/src/test/results/clientpositive/perf/query67.q.out index 1f38027..2831261 100644 --- ql/src/test/results/clientpositive/perf/query67.q.out +++ ql/src/test/results/clientpositive/perf/query67.q.out @@ -87,93 +87,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_35] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_30] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_47] (rows=1149975358 width=88) - predicate:(rank_window_0 <= 100) - PTF Operator [PTF_29] (rows=3449926075 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col16 DESC NULLS LAST","partition by:":"_col0"}] - Select Operator [SEL_28] (rows=3449926075 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col16"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Select Operator [SEL_26] (rows=3449926075 width=88) - Output:["_col0","_col1","_col16","_col2","_col3","_col4","_col5","_col6","_col7"] - Group By Operator [GBY_25] (rows=3449926075 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Group By Operator [GBY_23] (rows=6899852151 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, 0 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_54] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col4","_col7","_col8","_col9","_col11","_col13","_col14","_col15","_col16"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_51] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category","i_product_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_53] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3","_col4","_col7","_col8","_col9","_col11"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_50] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_52] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col7","_col8","_col9"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_48] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_quantity","ss_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col2","_col3","_col4"] - Filter Operator [FIL_49] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq","d_year","d_moy","d_qoy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query68.q.out ql/src/test/results/clientpositive/perf/query68.q.out index b84bdf6..fb1b458 100644 --- ql/src/test/results/clientpositive/perf/query68.q.out +++ ql/src/test/results/clientpositive/perf/query68.q.out @@ -5,130 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 15 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 12 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 8 <- Map 13 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_50] - Limit [LIM_49] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_48] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Select Operator [SEL_46] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_45] (rows=463823414 width=88) - predicate:(_col5 <> _col8) - Merge Join Operator [MERGEJOIN_86] (rows=463823414 width=88) - Conds:RS_42._col0=RS_43._col1(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9","_col10","_col11"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_37] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_36] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_34] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col1, _col18, _col3, _col5 - Select Operator [SEL_33] (rows=843315281 width=88) - Output:["_col1","_col18","_col3","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_85] (rows=843315281 width=88) - Conds:RS_30._col3=RS_31._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_18] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=766650239 width=88) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=7200 width=107) - Output:["_col0"] - Filter Operator [FIL_79] (rows=7200 width=107) - predicate:(((hd_dep_count = 4) or (hd_vehicle_count = 2)) and hd_demo_sk is not null) - TableScan [TS_15] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=696954748 width=88) - Conds:RS_24._col4=RS_25._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_78] (rows=852 width=1910) - predicate:((s_city) IN ('Rosedale', 'Bethlehem') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_city"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_82] (rows=633595212 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_77] (rows=4058 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2 and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Filter Operator [FIL_76] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null and ss_customer_sk is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_ext_sales_price","ss_ext_list_price","ss_ext_tax"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=88000001 width=860) - Conds:RS_39._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col3","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_74] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,current_addr,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query69.q.out ql/src/test/results/clientpositive/perf/query69.q.out index 71ddd71..0bd5290 100644 --- ql/src/test/results/clientpositive/perf/query69.q.out +++ ql/src/test/results/clientpositive/perf/query69.q.out @@ -1,12 +1,12 @@ -Warning: Shuffle Join MERGEJOIN[261][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 17' is a cross product -Warning: Shuffle Join MERGEJOIN[262][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 22' is a cross product -Warning: Shuffle Join MERGEJOIN[273][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Reducer 13' is a cross product -Warning: Shuffle Join MERGEJOIN[263][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 33' is a cross product -Warning: Shuffle Join MERGEJOIN[264][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 38' is a cross product -Warning: Shuffle Join MERGEJOIN[275][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_0]] in Stage 'Reducer 29' is a cross product -Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 49' is a cross product -Warning: Shuffle Join MERGEJOIN[266][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 54' is a cross product -Warning: Shuffle Join MERGEJOIN[278][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Reducer 45' is a cross product +Warning: Shuffle Join MERGEJOIN[261][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 17' is a cross product +Warning: Shuffle Join MERGEJOIN[262][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 22' is a cross product +Warning: Shuffle Join MERGEJOIN[273][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Vertex 13' is a cross product +Warning: Shuffle Join MERGEJOIN[263][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 33' is a cross product +Warning: Shuffle Join MERGEJOIN[264][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Vertex 38' is a cross product +Warning: Shuffle Join MERGEJOIN[275][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_0]] in Stage 'Vertex 29' is a cross product +Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Vertex 49' is a cross product +Warning: Shuffle Join MERGEJOIN[266][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Vertex 54' is a cross product +Warning: Shuffle Join MERGEJOIN[278][tables = [$hdt$_3, $hdt$_4, $hdt$_2, $hdt$_1]] in Stage 'Vertex 45' is a cross product PREHOOK: query: explain select cd_gender, cd_marital_status, @@ -100,396 +100,40 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 23 (CUSTOM_SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (CUSTOM_SIMPLE_EDGE), Map 19 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (CUSTOM_SIMPLE_EDGE), Map 24 (CUSTOM_SIMPLE_EDGE), Map 25 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 39 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (CUSTOM_SIMPLE_EDGE), Map 35 (CUSTOM_SIMPLE_EDGE), Map 36 (CUSTOM_SIMPLE_EDGE) -Reducer 34 <- Reducer 33 (SIMPLE_EDGE) -Reducer 38 <- Map 37 (CUSTOM_SIMPLE_EDGE), Map 40 (CUSTOM_SIMPLE_EDGE), Map 41 (CUSTOM_SIMPLE_EDGE) -Reducer 39 <- Reducer 38 (SIMPLE_EDGE) -Reducer 4 <- Reducer 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 47 (SIMPLE_EDGE) -Reducer 44 <- Reducer 43 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE) -Reducer 45 <- Reducer 44 (CUSTOM_SIMPLE_EDGE), Reducer 55 (CUSTOM_SIMPLE_EDGE) -Reducer 46 <- Reducer 45 (SIMPLE_EDGE) -Reducer 49 <- Map 48 (CUSTOM_SIMPLE_EDGE), Map 51 (CUSTOM_SIMPLE_EDGE), Map 52 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 46 (SIMPLE_EDGE) -Reducer 50 <- Reducer 49 (SIMPLE_EDGE) -Reducer 54 <- Map 53 (CUSTOM_SIMPLE_EDGE), Map 56 (CUSTOM_SIMPLE_EDGE), Map 57 (CUSTOM_SIMPLE_EDGE) -Reducer 55 <- Reducer 54 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (PARTITION_EDGE), Vertex 23 (PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (PARTITION_EDGE), Vertex 19 (PARTITION_EDGE), Vertex 20 (PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 24 (PARTITION_EDGE), Vertex 25 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (PARTITION_EDGE), Vertex 39 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (PARTITION_EDGE), Vertex 35 (PARTITION_EDGE), Vertex 36 (PARTITION_EDGE) +Vertex 34 <- Vertex 33 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (PARTITION_EDGE), Vertex 40 (PARTITION_EDGE), Vertex 41 (PARTITION_EDGE) +Vertex 39 <- Vertex 38 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 50 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (PARTITION_EDGE), Vertex 55 (PARTITION_EDGE) +Vertex 46 <- Vertex 45 (SORT_PARTITION_EDGE) +Vertex 49 <- Vertex 48 (PARTITION_EDGE), Vertex 51 (PARTITION_EDGE), Vertex 52 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 50 <- Vertex 49 (SORT_PARTITION_EDGE) +Vertex 54 <- Vertex 53 (PARTITION_EDGE), Vertex 56 (PARTITION_EDGE), Vertex 57 (PARTITION_EDGE) +Vertex 55 <- Vertex 54 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_184] - Limit [LIM_183] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_182] (rows=1268213682555322368 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_181] - Select Operator [SEL_180] (rows=1268213682555322368 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col6"] - Group By Operator [GBY_179] (rows=1268213682555322368 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_178] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_177] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count()"],keys:_col6, _col7, _col8, _col9, _col10 - Select Operator [SEL_176] (rows=2536427365110644736 width=1) - Output:["_col6","_col7","_col8","_col9","_col10"] - Filter Operator [FIL_175] (rows=2536427365110644736 width=1) - predicate:_col15 is null - Merge Join Operator [MERGEJOIN_279] (rows=5072854730221289472 width=1) - Conds:RS_172._col0=RS_173._col0(Left Outer),Output:["_col6","_col7","_col8","_col9","_col10","_col15"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Select Operator [SEL_121] (rows=4611686018427387903 width=1) - Output:["_col0","_col10","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_120] (rows=4611686018427387903 width=1) - predicate:_col13 is null - Select Operator [SEL_119] (rows=9223372036854775807 width=1) - Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col13"] - Merge Join Operator [MERGEJOIN_276] (rows=9223372036854775807 width=1) - Conds:RS_115._col0=RS_116._col0(Left Outer),RS_115._col0=RS_117._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_58] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_57] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Group By Operator [GBY_55] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_273] (rows=9223372036854775807 width=1) - Conds:(Inner),Output:["_col6"] - <-Reducer 12 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_51] - Merge Join Operator [MERGEJOIN_272] (rows=5072854730221289472 width=1) - Conds:RS_48._col1=RS_49._col0(Inner) - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_269] (rows=158402938 width=135) - Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_249] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_9] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_250] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_26] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_261] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 16 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_21] - Select Operator [SEL_16] (rows=40000000 width=4) - TableScan [TS_15] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_22] - Select Operator [SEL_18] (rows=1861800 width=4) - TableScan [TS_17] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Select Operator [SEL_20] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_19] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 23 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Group By Operator [GBY_43] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Group By Operator [GBY_41] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_262] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_36] - Select Operator [SEL_31] (rows=40000000 width=4) - TableScan [TS_30] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 24 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_37] - Select Operator [SEL_33] (rows=1861800 width=4) - TableScan [TS_32] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 25 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_38] - Select Operator [SEL_35] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_34] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_268] (rows=96800003 width=860) - Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1861800 width=385) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_248] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_6] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status","cd_purchase_estimate","cd_credit_rating"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_267] (rows=88000001 width=860) - Conds:RS_109._col2=RS_110._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_246] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_247] (rows=20000000 width=1014) - predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_117] - PartitionCols:_col0 - Group By Operator [GBY_107] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col0 - Group By Operator [GBY_105] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_275] (rows=9223372036854775807 width=1) - Conds:(Inner),Output:["_col6"] - <-Reducer 28 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_101] - Merge Join Operator [MERGEJOIN_274] (rows=5072854730221289472 width=1) - Conds:RS_98._col1=RS_99._col0(Inner) - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_270] (rows=633595212 width=88) - Conds:RS_95._col0=RS_96._col0(Inner),Output:["_col1"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Select Operator [SEL_61] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_253] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_59] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0 - Select Operator [SEL_64] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_254] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_62] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 34 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0 - Group By Operator [GBY_78] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Group By Operator [GBY_76] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_263] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 32 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_66] (rows=40000000 width=4) - TableScan [TS_65] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 35 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_72] - Select Operator [SEL_68] (rows=1861800 width=4) - TableScan [TS_67] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 36 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_73] - Select Operator [SEL_70] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_69] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 39 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_102] - Group By Operator [GBY_93] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col0 - Group By Operator [GBY_91] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_264] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 37 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_86] - Select Operator [SEL_81] (rows=40000000 width=4) - TableScan [TS_80] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 40 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_87] - Select Operator [SEL_83] (rows=1861800 width=4) - TableScan [TS_82] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 41 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_88] - Select Operator [SEL_85] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_84] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 46 [SIMPLE_EDGE] - SHUFFLE [RS_173] - PartitionCols:_col0 - Select Operator [SEL_171] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_170] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 45 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col0 - Group By Operator [GBY_168] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col6 - Merge Join Operator [MERGEJOIN_278] (rows=9223372036854775807 width=1) - Conds:(Inner),Output:["_col6"] - <-Reducer 44 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_164] - Merge Join Operator [MERGEJOIN_277] (rows=5072854730221289472 width=1) - Conds:RS_161._col1=RS_162._col0(Inner) - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_161] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_271] (rows=316788826 width=135) - Conds:RS_158._col0=RS_159._col0(Inner),Output:["_col1"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_158] - PartitionCols:_col0 - Select Operator [SEL_124] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_257] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_122] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_ship_customer_sk"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_159] - PartitionCols:_col0 - Select Operator [SEL_127] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_258] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - TableScan [TS_125] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 50 [SIMPLE_EDGE] - SHUFFLE [RS_162] - PartitionCols:_col0 - Group By Operator [GBY_141] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 49 [SIMPLE_EDGE] - SHUFFLE [RS_140] - PartitionCols:_col0 - Group By Operator [GBY_139] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_265] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 48 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_134] - Select Operator [SEL_129] (rows=40000000 width=4) - TableScan [TS_128] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 51 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_135] - Select Operator [SEL_131] (rows=1861800 width=4) - TableScan [TS_130] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 52 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_136] - Select Operator [SEL_133] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_132] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 55 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_165] - Group By Operator [GBY_156] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 54 [SIMPLE_EDGE] - SHUFFLE [RS_155] - PartitionCols:_col0 - Group By Operator [GBY_154] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_266] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 53 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_149] - Select Operator [SEL_144] (rows=40000000 width=4) - TableScan [TS_143] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:COMPLETE - <-Map 56 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_150] - Select Operator [SEL_146] (rows=1861800 width=4) - TableScan [TS_145] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE - <-Map 57 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_151] - Select Operator [SEL_148] (rows=80000000 width=860) - Output:["_col0"] - TableScan [TS_147] (rows=80000000 width=860) - default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query7.q.out ql/src/test/results/clientpositive/perf/query7.q.out index 65478ab..18ae02d 100644 --- ql/src/test/results/clientpositive/perf/query7.q.out +++ ql/src/test/results/clientpositive/perf/query7.q.out @@ -5,94 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_35] - Limit [LIM_34] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_33] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_32] - Group By Operator [GBY_30] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","avg(VALUE._col3)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_28] (rows=843315281 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["avg(_col4)","avg(_col5)","avg(_col7)","avg(_col6)"],keys:_col18 - Select Operator [SEL_27] (rows=843315281 width=88) - Output:["_col18","_col4","_col5","_col7","_col6"] - Merge Join Operator [MERGEJOIN_58] (rows=843315281 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col7","_col18"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_54] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_12] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_57] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_53] (rows=2300 width=1179) - predicate:(((p_channel_email = 'N') or (p_channel_event = 'N')) and p_promo_sk is not null) - TableScan [TS_9] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_email","p_channel_event"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_56] (rows=696954748 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_52] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_55] (rows=633595212 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_50] (rows=575995635 width=88) - predicate:(ss_cdemo_sk is not null and ss_sold_date_sk is not null and ss_item_sk is not null and ss_promo_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_cdemo_sk","ss_promo_sk","ss_quantity","ss_list_price","ss_sales_price","ss_coupon_amt"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=232725 width=385) - Output:["_col0"] - Filter Operator [FIL_51] (rows=232725 width=385) - predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query70.q.out ql/src/test/results/clientpositive/perf/query70.q.out index d0900a8..b1e4711 100644 --- ql/src/test/results/clientpositive/perf/query70.q.out +++ ql/src/test/results/clientpositive/perf/query70.q.out @@ -75,151 +75,21 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 12 <- Map 17 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 15 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_64] - Limit [LIM_63] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_62] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_61] - Select Operator [SEL_59] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_58] (rows=1149975358 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_57] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_55] (rows=1149975358 width=88) - Output:["_col0","_col1","_col4","_col5"] - Group By Operator [GBY_54] (rows=1149975358 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_52] (rows=2299950717 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col0, _col1, 0 - Select Operator [SEL_50] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_92] (rows=766650239 width=88) - Conds:RS_47._col7=RS_48._col0(Inner),Output:["_col2","_col6","_col7"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Group By Operator [GBY_39] (rows=58079562 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Group By Operator [GBY_37] (rows=116159124 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_32] (rows=116159124 width=88) - Output:["_col0"] - Filter Operator [FIL_84] (rows=116159124 width=88) - predicate:(rank_window_0 <= 5) - PTF Operator [PTF_31] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"_col0"}] - Select Operator [SEL_30] (rows=348477374 width=88) - Output:["_col0","_col1"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col6 - Select Operator [SEL_24] (rows=696954748 width=88) - Output:["_col6","_col2"] - Merge Join Operator [MERGEJOIN_91] (rows=696954748 width=88) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col6"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_87] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_state is not null) - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_90] (rows=633595212 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_85] (rows=575995635 width=88) - predicate:(ss_store_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_9] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_86] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_89] (rows=696954748 width=88) - Conds:RS_44._col1=RS_45._col0(Inner),Output:["_col2","_col6","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_83] (rows=1704 width=1910) - predicate:(s_state is not null and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,s,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county","s_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_88] (rows=633595212 width=88) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_81] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_82] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query71.q.out ql/src/test/results/clientpositive/perf/query71.q.out index 9bb010c..cce17b4 100644 --- ql/src/test/results/clientpositive/perf/query71.q.out +++ ql/src/test/results/clientpositive/perf/query71.q.out @@ -5,134 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Map 15 (SIMPLE_EDGE), Union 3 (SIMPLE_EDGE) -Reducer 5 <- Map 16 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 10 <- Union 3 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 3 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Union 3 (CONTAINS), Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 3 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_53] - Select Operator [SEL_52] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_51] - Select Operator [SEL_49] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_48] (rows=670816149 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_46] (rows=1341632299 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col0)"],keys:_col4, _col8, _col9, _col5 - Select Operator [SEL_45] (rows=1341632299 width=108) - Output:["_col4","_col8","_col9","_col5","_col0"] - Merge Join Operator [MERGEJOIN_87] (rows=1341632299 width=108) - Conds:RS_42._col2=RS_43._col0(Inner),Output:["_col0","_col4","_col5","_col8","_col9"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=86400 width=471) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=86400 width=471) - predicate:(((t_meal_time = 'breakfast') or (t_meal_time = 'dinner')) and t_time_sk is not null) - TableScan [TS_36] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute","t_meal_time"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_86] (rows=1219665700 width=108) - Conds:Union 3._col1=RS_40._col0(Inner),Output:["_col0","_col2","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col0 - Select Operator [SEL_35] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_81] (rows=231000 width=1436) - predicate:((i_manager_id = 1) and i_item_sk is not null) - TableScan [TS_33] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_brand","i_manager_id"] - <-Union 3 [SIMPLE_EDGE] - <-Reducer 10 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_84] (rows=316788826 width=135) - Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_78] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_12] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_77] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_item_sk is not null and cs_sold_time_sk is not null) - TableScan [TS_10] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_31] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_85] (rows=633595212 width=88) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_79] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null and ss_sold_time_sk is not null) - TableScan [TS_22] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_80] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [CONTAINS] - Reduce Output Operator [RS_39] - PartitionCols:_col1 - Select Operator [SEL_9] (rows=158402938 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_83] (rows=158402938 width=135) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_75] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_item_sk is not null and ws_sold_time_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_item_sk","ws_ext_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=18262 width=1119) - predicate:((d_moy = 12) and (d_year = 2001) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query72.q.out ql/src/test/results/clientpositive/perf/query72.q.out index 828aa28..45c92f6 100644 --- ql/src/test/results/clientpositive/perf/query72.q.out +++ ql/src/test/results/clientpositive/perf/query72.q.out @@ -61,192 +61,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 22 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 23 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) -Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 18 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 20 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 21 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 13 - File Output Operator [FS_76] - Limit [LIM_75] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_74] (rows=41498422 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_73] - Group By Operator [GBY_71] (rows=41498422 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_69] (rows=82996844 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col3)","count(_col4)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_67] (rows=82996844 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Merge Join Operator [MERGEJOIN_143] (rows=82996844 width=135) - Conds:RS_64._col4, _col6=RS_65._col0, _col1(Left Outer),Output:["_col13","_col15","_col22","_col28"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Select Operator [SEL_60] (rows=28798881 width=106) - Output:["_col0","_col1"] - Filter Operator [FIL_133] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_58] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col4, _col6 - Merge Join Operator [MERGEJOIN_142] (rows=75451675 width=135) - Conds:RS_61._col5=RS_62._col0(Left Outer),Output:["_col4","_col6","_col13","_col15","_col22","_col28"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=2300 width=1179) - Output:["_col0"] - TableScan [TS_56] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col5 - Select Operator [SEL_55] (rows=68592431 width=135) - Output:["_col13","_col15","_col22","_col4","_col5","_col6"] - Filter Operator [FIL_54] (rows=68592431 width=135) - predicate:(UDFToDouble(_col27) > (UDFToDouble(_col21) + 5.0)) - Select Operator [SEL_53] (rows=205777294 width=135) - Output:["_col4","_col5","_col6","_col13","_col15","_col21","_col22","_col27"] - Merge Join Operator [MERGEJOIN_141] (rows=205777294 width=135) - Conds:RS_50._col10=RS_51._col0(Inner),Output:["_col4","_col5","_col6","_col13","_col14","_col23","_col25","_col27"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_131] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_29] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col10 - Merge Join Operator [MERGEJOIN_140] (rows=187070264 width=135) - Conds:RS_47._col1=RS_48._col0(Inner),Output:["_col4","_col5","_col6","_col10","_col13","_col14","_col23","_col25"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_130] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_26] (rows=73049 width=1119) - default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_139] (rows=170063873 width=135) - Conds:RS_44._col4=RS_45._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col10","_col13","_col14","_col23"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_129] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_23] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_138] (rows=154603518 width=135) - Conds:RS_41._col3=RS_42._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_128] (rows=3600 width=107) - predicate:((hd_buy_potential = '1001-5000') and hd_demo_sk is not null) - TableScan [TS_20] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_137] (rows=140548650 width=135) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=930900 width=385) - Output:["_col0"] - Filter Operator [FIL_127] (rows=930900 width=385) - predicate:((cd_marital_status = 'M') and cd_demo_sk is not null) - TableScan [TS_17] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_136] (rows=127771498 width=135) - Conds:RS_35._col8, _col14=RS_36._col0, _col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col10","_col13","_col14"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Select Operator [SEL_16] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_week_seq is not null) - TableScan [TS_14] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col8, _col14 - Merge Join Operator [MERGEJOIN_135] (rows=116155905 width=135) - Conds:RS_32._col0=RS_33._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col8","_col10","_col13","_col14"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_13] (rows=36524 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_125] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null and d_week_seq is not null) - TableScan [TS_11] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_week_seq","d_year"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_10] (rows=105596275 width=135) - Output:["_col0","_col1","_col10","_col2","_col3","_col4","_col5","_col6","_col8"] - Filter Operator [FIL_9] (rows=105596275 width=135) - predicate:(_col11 < _col7) - Merge Join Operator [MERGEJOIN_134] (rows=316788826 width=135) - Conds:RS_6._col4=RS_7._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col10","_col11"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col4 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_123] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_bill_cdemo_sk is not null and cs_bill_hdemo_sk is not null and cs_sold_date_sk is not null and cs_ship_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_ship_date_sk","cs_bill_cdemo_sk","cs_bill_hdemo_sk","cs_item_sk","cs_promo_sk","cs_order_number","cs_quantity"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_124] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - TableScan [TS_3] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] + Vertex 13 diff --git ql/src/test/results/clientpositive/perf/query73.q.out ql/src/test/results/clientpositive/perf/query73.q.out index 9e1a39a..5daf3d9 100644 --- ql/src/test/results/clientpositive/perf/query73.q.out +++ ql/src/test/results/clientpositive/perf/query73.q.out @@ -5,96 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=88000001 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=88000001 width=860) - Conds:RS_31._col0=RS_32._col1(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_salutation","c_first_name","c_last_name","c_preferred_cust_flag"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col1 - Filter Operator [FIL_29] (rows=42591679 width=88) - predicate:_col2 BETWEEN 1 AND 5 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col4 - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col3=RS_22._col0(Inner),Output:["_col1","_col4"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_56] (rows=852 width=1910) - predicate:((s_county) IN ('Kittitas County', 'Adams County', 'Richland County', 'Furnas County') and s_store_sk is not null) - TableScan [TS_12] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1200 width=107) - Output:["_col0"] - Filter Operator [FIL_55] (rows=1200 width=107) - predicate:(((hd_buy_potential = '1001-5000') or (hd_buy_potential = '5001-10000')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.0)) ELSE (null) END and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential","hd_dep_count","hd_vehicle_count"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_store_sk","ss_ticket_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=4058 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and d_dom BETWEEN 1 AND 2 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dom"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query75.q.out ql/src/test/results/clientpositive/perf/query75.q.out index f78ce8a..b93f79e 100644 --- ql/src/test/results/clientpositive/perf/query75.q.out +++ ql/src/test/results/clientpositive/perf/query75.q.out @@ -5,387 +5,32 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 20 <- Map 19 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 21 <- Map 24 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Map 25 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE) -Reducer 28 <- Map 33 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 34 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 31 <- Union 30 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 37 <- Map 40 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Map 41 (SIMPLE_EDGE), Reducer 37 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 44 <- Map 47 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 45 <- Map 48 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE), Union 30 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 31 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 5 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 22 <- Union 5 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 29 <- Union 30 (CONTAINS), Vertex 28 (SORT_PARTITION_EDGE), Vertex 34 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 31 <- Union 30 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 38 <- Union 30 (CONTAINS), Vertex 37 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 46 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 45 <- Union 30 (CONTAINS), Vertex 44 (SORT_PARTITION_EDGE), Vertex 48 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 8 - File Output Operator [FS_156] - Limit [LIM_155] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_154] (rows=245965926 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_153] - Select Operator [SEL_152] (rows=245965926 width=108) - Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_151] (rows=245965926 width=108) - predicate:((CAST( _col4 AS decimal(17,2)) / CAST( _col10 AS decimal(17,2))) < 0.9) - Merge Join Operator [MERGEJOIN_259] (rows=737897778 width=108) - Conds:RS_148._col0, _col1, _col2, _col3=RS_149._col0, _col1, _col2, _col3(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col10","_col11"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_146] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Union 30 [SIMPLE_EDGE] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_95] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_252] (rows=383314495 width=135) - Conds:RS_92._col1, _col2=RS_93._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_232] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_83] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_251] (rows=348467716 width=135) - Conds:RS_89._col1=RS_90._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_82] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_231] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_80] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_250] (rows=316788826 width=135) - Conds:RS_86._col0=RS_87._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0 - Select Operator [SEL_76] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_229] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_74] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col0 - Select Operator [SEL_79] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_230] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_77] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 38 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_117] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_255] (rows=766650239 width=88) - Conds:RS_114._col1, _col2=RS_115._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_236] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_105] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_114] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_254] (rows=696954748 width=88) - Conds:RS_111._col1=RS_112._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Select Operator [SEL_104] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_235] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_102] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_253] (rows=633595212 width=88) - Conds:RS_108._col0=RS_109._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_98] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_233] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_96] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_234] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_99] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 45 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_141] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_258] (rows=191667562 width=135) - Conds:RS_138._col1, _col2=RS_139._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 48 [SIMPLE_EDGE] - SHUFFLE [RS_139] - PartitionCols:_col0, _col1 - Select Operator [SEL_131] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_240] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_129] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_138] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_257] (rows=174243235 width=135) - Conds:RS_135._col1=RS_136._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_136] - PartitionCols:_col0 - Select Operator [SEL_128] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_239] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_126] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_135] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_256] (rows=158402938 width=135) - Conds:RS_132._col0=RS_133._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col0 - Select Operator [SEL_122] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_237] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_120] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Select Operator [SEL_125] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_238] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_123] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_72] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_43] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_246] (rows=766650239 width=88) - Conds:RS_40._col1, _col2=RS_41._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0, _col1 - Select Operator [SEL_33] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_224] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_31] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_245] (rows=696954748 width=88) - Conds:RS_37._col1=RS_38._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_30] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_223] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_28] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_244] (rows=633595212 width=88) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_221] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_22] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_222] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 22 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_67] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_249] (rows=191667562 width=135) - Conds:RS_64._col1, _col2=RS_65._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Select Operator [SEL_57] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_228] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_55] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_248] (rows=174243235 width=135) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_227] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_52] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_247] (rows=158402938 width=135) - Conds:RS_58._col0=RS_59._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_225] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_46] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_226] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_49] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Select Operator [SEL_21] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_243] (rows=383314495 width=135) - Conds:RS_18._col1, _col2=RS_19._col0, _col1(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_11] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_220] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_9] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1, _col2 - Merge Join Operator [MERGEJOIN_242] (rows=348467716 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_219] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_241] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_217] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_218] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Vertex 8 diff --git ql/src/test/results/clientpositive/perf/query76.q.out ql/src/test/results/clientpositive/perf/query76.q.out index c7dbb37..8f1d85c 100644 --- ql/src/test/results/clientpositive/perf/query76.q.out +++ ql/src/test/results/clientpositive/perf/query76.q.out @@ -5,151 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 13 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) -Reducer 16 <- Map 18 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Union 4 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 4 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 4 (CONTAINS), Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_59] - Limit [LIM_58] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_57] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Group By Operator [GBY_54] (rows=304916424 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Union 4 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_31] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_87] (rows=87121617 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col3","_col5","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_80] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_22] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_86] (rows=79201469 width=135) - Conds:RS_25._col1=RS_26._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Select Operator [SEL_21] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_79] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_19] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col1 - Select Operator [SEL_18] (rows=72001334 width=135) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_78] (rows=72001334 width=135) - predicate:(ws_web_page_sk is null and ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_16] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_page_sk","ws_ext_sales_price"] - <-Reducer 16 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_49] (rows=174233858 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_89] (rows=174233858 width=135) - Conds:RS_46._col0=RS_47._col0(Inner),Output:["_col3","_col5","_col7","_col8"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_83] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_40] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_88] (rows=158394413 width=135) - Conds:RS_43._col2=RS_44._col0(Inner),Output:["_col0","_col3","_col5"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col2 - Select Operator [SEL_36] (rows=143994918 width=135) - Output:["_col0","_col2","_col3"] - Filter Operator [FIL_81] (rows=143994918 width=135) - predicate:(cs_warehouse_sk is null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_34] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_warehouse_sk","cs_item_sk","cs_ext_sales_price"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Select Operator [SEL_39] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_82] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_37] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Reducer 3 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_52] (rows=609832848 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["count()","sum(_col5)"],keys:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_15] (rows=348477373 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_85] (rows=348477373 width=88) - Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col1","_col5","_col7","_col8"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_77] (rows=73049 width=1119) - predicate:d_date_sk is not null - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_84] (rows=316797605 width=88) - Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col1","_col2","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=287997817 width=88) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_76] (rows=287997817 width=88) - predicate:(ss_addr_sk is null and ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query79.q.out ql/src/test/results/clientpositive/perf/query79.q.out index 6ce9592..fc79760 100644 --- ql/src/test/results/clientpositive/perf/query79.q.out +++ ql/src/test/results/clientpositive/perf/query79.q.out @@ -5,98 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_35] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=421657640 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_60] (rows=421657640 width=88) - Conds:RS_30._col0=RS_31._col1(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_52] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_28] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1, _col2, _col3 - Group By Operator [GBY_25] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col6)","sum(_col7)"],keys:_col1, _col3, _col5, _col13 - Select Operator [SEL_24] (rows=766650239 width=88) - Output:["_col1","_col3","_col5","_col13","_col6","_col7"] - Merge Join Operator [MERGEJOIN_59] (rows=766650239 width=88) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col13"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_56] (rows=6000 width=107) - predicate:(((hd_dep_count = 8) or (hd_vehicle_count > 0)) and hd_demo_sk is not null) - TableScan [TS_12] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_58] (rows=696954748 width=88) - Conds:RS_18._col4=RS_19._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col7","_col13"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=189 width=1910) - Output:["_col0","_col2"] - Filter Operator [FIL_55] (rows=189 width=1910) - predicate:(s_number_employees BETWEEN 200 AND 295 and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_number_employees","s_city"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_57] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_53] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_hdemo_sk is not null and ss_customer_sk is not null) - TableScan [TS_3] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_ticket_number","ss_coupon_amt","ss_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_54] (rows=18262 width=1119) - predicate:((d_year) IN (1998, 1999, 2000) and (d_dow = 1) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_dow"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query8.q.out ql/src/test/results/clientpositive/perf/query8.q.out index 5642336..fdd39bf 100644 --- ql/src/test/results/clientpositive/perf/query8.q.out +++ ql/src/test/results/clientpositive/perf/query8.q.out @@ -215,146 +215,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Union 9 (SIMPLE_EDGE) -Reducer 11 <- Map 17 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Union 9 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 9 (CONTAINS) +Vertex 10 <- Union 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 9 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 9 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_63] - Limit [LIM_62] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_61] (rows=348477374 width=88) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Group By Operator [GBY_58] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Group By Operator [GBY_56] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col8 - Select Operator [SEL_55] (rows=696954748 width=88) - Output:["_col8","_col2"] - Merge Join Operator [MERGEJOIN_89] (rows=696954748 width=88) - Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2","_col8"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_88] (rows=1874 width=1911) - Conds:RS_45.substr(_col0, 1, 2)=RS_46.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:substr(_col2, 1, 2) - Select Operator [SEL_44] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_85] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_zip is not null) - TableScan [TS_42] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:substr(_col0, 1, 2) - Select Operator [SEL_41] (rows=1 width=1014) - Output:["_col0"] - Filter Operator [FIL_40] (rows=1 width=1014) - predicate:(_col1 = 2) - Group By Operator [GBY_39] (rows=6833333 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Union 9 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_38] - PartitionCols:_col0 - Group By Operator [GBY_37] (rows=13666666 width=1014) - Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 - Group By Operator [GBY_32] (rows=3666666 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Group By Operator [GBY_30] (rows=7333333 width=1014) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_28] (rows=7333333 width=1014) - Output:["_col0"] - Filter Operator [FIL_27] (rows=7333333 width=1014) - predicate:(_col1 > 10) - Group By Operator [GBY_26] (rows=22000000 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=44000000 width=1014) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col1 - Merge Join Operator [MERGEJOIN_87] (rows=44000000 width=1014) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_83] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and substr(ca_zip, 1, 5) is not null) - TableScan [TS_14] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_zip"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=40000000 width=860) - Output:["_col0"] - Filter Operator [FIL_84] (rows=40000000 width=860) - predicate:((c_preferred_cust_flag = 'Y') and c_current_addr_sk is not null) - TableScan [TS_17] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_current_addr_sk","c_preferred_cust_flag"] - <-Reducer 8 [CONTAINS] - Reduce Output Operator [RS_38] - PartitionCols:_col0 - Group By Operator [GBY_37] (rows=13666666 width=1014) - Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 - Group By Operator [GBY_12] (rows=10000000 width=1014) - Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0 - Group By Operator [GBY_10] (rows=20000000 width=1014) - Output:["_col0","_col1"],aggregations:["count(1)"],keys:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_81] (rows=20000000 width=1014) - predicate:(substr(ca_zip, 1, 5)) IN ('89436', '30868', '65085', '22977', '83927', '77557', '58429', '40697', '80614', '10502', '32779', '91137', '61265', '98294', '17921', '18427', '21203', '59362', '87291', '84093', '21505', '17184', '10866', '67898', '25797', '28055', '18377', '80332', '74535', '21757', '29742', '90885', '29898', '17819', '40811', '25990', '47513', '89531', '91068', '10391', '18846', '99223', '82637', '41368', '83658', '86199', '81625', '26696', '89338', '88425', '32200', '81427', '19053', '77471', '36610', '99823', '43276', '41249', '48584', '83550', '82276', '18842', '78890', '14090', '38123', '40936', '34425', '19850', '43286', '80072', '79188', '54191', '11395', '50497', '84861', '90733', '21068', '57666', '37119', '25004', '57835', '70067', '62878', '95806', '19303', '18840', '19124', '29785', '16737', '16022', '49613', '89977', '68310', '60069', '98360', '48649', '39050', '41793', '25002', '27413', '39736', '47208', '16515', '94808', '57648', '15009', '80015', '42961', '63982', '21744', '71853', '81087', '67468', '34175', '64008', '20261', '11201', '51799', '48043', '45645', '61163', '48375', '36447', '57042', '21218', '41100', '89951', '22745', '35851', '83326', '61125', '78298', '80752', '49858', '52940', '96976', '63792', '11376', '53582', '18717', '90226', '50530', '94203', '99447', '27670', '96577', '57856', '56372', '16165', '23427', '54561', '28806', '44439', '22926', '30123', '61451', '92397', '56979', '92309', '70873', '13355', '21801', '46346', '37562', '56458', '28286', '47306', '99555', '69399', '26234', '47546', '49661', '88601', '35943', '39936', '25632', '24611', '44166', '56648', '30379', '59785', '11110', '14329', '93815', '52226', '71381', '13842', '25612', '63294', '14664', '21077', '82626', '18799', '60915', '81020', '56447', '76619', '11433', '13414', '42548', '92713', '70467', '30884', '47484', '16072', '38936', '13036', '88376', '45539', '35901', '19506', '65690', '73957', '71850', '49231', '14276', '20005', '18384', '76615', '11635', '38177', '55607', '41369', '95447', '58581', '58149', '91946', '33790', '76232', '75692', '95464', '22246', '51061', '56692', '53121', '77209', '15482', '10688', '14868', '45907', '73520', '72666', '25734', '17959', '24677', '66446', '94627', '53535', '15560', '41967', '69297', '11929', '59403', '33283', '52232', '57350', '43933', '40921', '36635', '10827', '71286', '19736', '80619', '25251', '95042', '15526', '36496', '55854', '49124', '81980', '35375', '49157', '63512', '28944', '14946', '36503', '54010', '18767', '23969', '43905', '66979', '33113', '21286', '58471', '59080', '13395', '79144', '70373', '67031', '38360', '26705', '50906', '52406', '26066', '73146', '15884', '31897', '30045', '61068', '45550', '92454', '13376', '14354', '19770', '22928', '97790', '50723', '46081', '30202', '14410', '20223', '88500', '67298', '13261', '14172', '81410', '93578', '83583', '46047', '94167', '82564', '21156', '15799', '86709', '37931', '74703', '83103', '23054', '70470', '72008', '49247', '91911', '69998', '20961', '70070', '63197', '54853', '88191', '91830', '49521', '19454', '81450', '89091', '62378', '25683', '61869', '51744', '36580', '85778', '36871', '48121', '28810', '83712', '45486', '67393', '26935', '42393', '20132', '55349', '86057', '21309', '80218', '10094', '11357', '48819', '39734', '40758', '30432', '21204', '29467', '30214', '61024', '55307', '74621', '11622', '68908', '33032', '52868', '99194', '99900', '84936', '69036', '99149', '45013', '32895', '59004', '32322', '14933', '32936', '33562', '72550', '27385', '58049', '58200', '16808', '21360', '32961', '18586', '79307', '15492') - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_zip"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_86] (rows=633595212 width=88) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_79] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query80.q.out ql/src/test/results/clientpositive/perf/query80.q.out index 7338d9c..3f73b54 100644 --- ql/src/test/results/clientpositive/perf/query80.q.out +++ ql/src/test/results/clientpositive/perf/query80.q.out @@ -191,318 +191,30 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 18 <- Map 24 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 25 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE) -Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 27 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 29 <- Map 28 (SIMPLE_EDGE), Map 35 (SIMPLE_EDGE) -Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Map 36 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE) -Reducer 31 <- Map 37 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 38 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 33 <- Map 39 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 34 <- Reducer 33 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 15 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 11 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 27 (SORT_PARTITION_EDGE) +Vertex 22 <- Union 8 (CONTAINS), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 35 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 32 <- Vertex 31 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 34 <- Union 8 (CONTAINS), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 8 (CONTAINS), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 10 - File Output Operator [FS_128] - Limit [LIM_127] (rows=100 width=108) - Number of rows:100 - Select Operator [SEL_126] (rows=1217531358 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_125] - Select Operator [SEL_124] (rows=1217531358 width=108) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_123] (rows=1217531358 width=108) - Output:["_col0","_col1","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 22 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_77] (rows=231905279 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_76] (rows=231905279 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Group By Operator [GBY_74] (rows=463810558 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_72] (rows=463810558 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_213] (rows=463810558 width=135) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Select Operator [SEL_56] (rows=46000 width=460) - Output:["_col0","_col1"] - Filter Operator [FIL_197] (rows=46000 width=460) - predicate:cp_catalog_page_sk is not null - TableScan [TS_54] (rows=46000 width=460) - default@catalog_page,catalog_page,Tbl:COMPLETE,Col:NONE,Output:["cp_catalog_page_sk","cp_catalog_page_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_212] (rows=421645953 width=135) - Conds:RS_66._col3=RS_67._col0(Inner),Output:["_col1","_col5","_col6","_col9","_col10"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_196] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_51] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_211] (rows=383314495 width=135) - Conds:RS_63._col2=RS_64._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col9","_col10"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_195] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_48] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_210] (rows=348467716 width=135) - Conds:RS_60._col0=RS_61._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_194] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - TableScan [TS_45] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_209] (rows=316788826 width=135) - Conds:RS_57._col2, _col4=RS_58._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col2, _col4 - Select Operator [SEL_41] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_192] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_catalog_page_sk is not null and cs_item_sk is not null and cs_promo_sk is not null) - TableScan [TS_39] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_catalog_page_sk","cs_item_sk","cs_promo_sk","cs_order_number","cs_ext_sales_price","cs_net_profit"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0, _col1 - Select Operator [SEL_44] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_193] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_42] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_amount","cr_net_loss"] - <-Reducer 34 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_118] (rows=115958879 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_117] (rows=115958879 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Group By Operator [GBY_115] (rows=231917759 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_113] (rows=231917759 width=135) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_218] (rows=231917759 width=135) - Conds:RS_110._col2=RS_111._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col0 - Select Operator [SEL_97] (rows=84 width=1850) - Output:["_col0","_col1"] - Filter Operator [FIL_203] (rows=84 width=1850) - predicate:web_site_sk is not null - TableScan [TS_95] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_site_id"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_217] (rows=210834322 width=135) - Conds:RS_107._col3=RS_108._col0(Inner),Output:["_col2","_col5","_col6","_col9","_col10"] - <-Map 38 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_94] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_202] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_92] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_216] (rows=191667562 width=135) - Conds:RS_104._col1=RS_105._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_91] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_201] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_89] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_215] (rows=174243235 width=135) - Conds:RS_101._col0=RS_102._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col0 - Select Operator [SEL_88] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_200] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - TableScan [TS_86] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_214] (rows=158402938 width=135) - Conds:RS_98._col1, _col4=RS_99._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col1, _col4 - Select Operator [SEL_82] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_198] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_web_site_sk is not null and ws_item_sk is not null and ws_promo_sk is not null) - TableScan [TS_80] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_site_sk","ws_promo_sk","ws_order_number","ws_ext_sales_price","ws_net_profit"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_99] - PartitionCols:_col0, _col1 - Select Operator [SEL_85] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_199] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_83] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_amt","wr_net_loss"] - <-Reducer 7 [CONTAINS] - Reduce Output Operator [RS_122] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_121] (rows=2435062716 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)"],keys:_col0, _col1, 0 - Select Operator [SEL_38] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_37] (rows=463823414 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0 - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Group By Operator [GBY_35] (rows=927646829 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)"],keys:_col0 - Select Operator [SEL_33] (rows=927646829 width=88) - Output:["_col0","_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_208] (rows=927646829 width=88) - Conds:RS_30._col2=RS_31._col0(Inner),Output:["_col5","_col6","_col9","_col10","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1704 width=1910) - Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_15] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_207] (rows=843315281 width=88) - Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col2","_col5","_col6","_col9","_col10"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_190] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - TableScan [TS_12] (rows=2300 width=1179) - default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_206] (rows=766650239 width=88) - Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_189] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - TableScan [TS_9] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_205] (rows=696954748 width=88) - Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_188] (rows=8116 width=1119) - predicate:(d_date BETWEEN 1998-08-04 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_204] (rows=633595212 width=88) - Conds:RS_18._col1, _col4=RS_19._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col9","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1, _col4 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_186] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null and ss_item_sk is not null and ss_promo_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_ext_sales_price","ss_net_profit"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_187] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_3] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_amt","sr_net_loss"] + Vertex 10 diff --git ql/src/test/results/clientpositive/perf/query81.q.out ql/src/test/results/clientpositive/perf/query81.q.out index 8ca0068..1810bcd 100644 --- ql/src/test/results/clientpositive/perf/query81.q.out +++ ql/src/test/results/clientpositive/perf/query81.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Reducer 22' is a cross product +Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_4, $hdt$_5, $hdt$_6]] in Stage 'Vertex 22' is a cross product PREHOOK: query: explain with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, @@ -60,249 +60,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) -Reducer 15 <- Map 20 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (CUSTOM_SIMPLE_EDGE), Map 30 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE) -Reducer 26 <- Map 29 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Reducer 26 (SIMPLE_EDGE) -Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 18 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Vertex 10 <- Vertex 9 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE), Vertex 27 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE) +Vertex 25 <- Vertex 24 (SORT_PARTITION_EDGE), Vertex 28 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE), Vertex 29 (SORT_PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_102] - Limit [LIM_101] (rows=100 width=1) - Number of rows:100 - Select Operator [SEL_100] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_99] - Select Operator [SEL_98] (rows=930023387364950016 width=1) - Output:["_col0","_col1","_col11","_col12","_col13","_col14","_col15","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Filter Operator [FIL_97] (rows=930023387364950016 width=1) - predicate:(_col2 > CASE WHEN (_col22 is null) THEN (null) ELSE (_col21) END) - Select Operator [SEL_96] (rows=2790070162094850048 width=1) - Output:["_col2","_col4","_col5","_col6","_col7","_col8","_col9","_col11","_col12","_col13","_col14","_col16","_col18","_col19","_col20","_col21","_col22"] - Merge Join Operator [MERGEJOIN_162] (rows=2790070162094850048 width=1) - Conds:RS_93._col19=RS_94._col2(Left Outer),Output:["_col1","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col14","_col15","_col16","_col17","_col20","_col21","_col22"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col2 - Select Operator [SEL_86] (rows=2536427365110644736 width=1) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_85] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_84] - PartitionCols:_col0 - Group By Operator [GBY_83] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col3 - Select Operator [SEL_82] (rows=5072854730221289472 width=1) - Output:["_col3","_col2"] - Merge Join Operator [MERGEJOIN_161] (rows=5072854730221289472 width=1) - Conds:RS_79._col1=RS_80._col0(Inner),Output:["_col2","_col3"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=22000000 width=1014) - Output:["_col1","_col2"] - Group By Operator [GBY_44] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1 - Group By Operator [GBY_42] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col7, _col1 - Select Operator [SEL_41] (rows=44000000 width=1014) - Output:["_col7","_col1","_col3"] - Merge Join Operator [MERGEJOIN_159] (rows=44000000 width=1014) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_32] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_158] (rows=31678769 width=106) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_146] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_returning_addr_sk is not null) - TableScan [TS_26] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_returning_addr_sk","cr_return_amt_inc_tax"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_147] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Group By Operator [GBY_77] (rows=4611686018427387903 width=1) - Output:["_col0"],keys:KEY._col0 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Group By Operator [GBY_75] (rows=9223372036854775807 width=1) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_154] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),Output:["_col2"] - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_70] - Select Operator [SEL_47] (rows=80000000 width=4) - TableScan [TS_46] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:COMPLETE - <-Map 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_72] - Select Operator [SEL_69] (rows=40000000 width=4) - TableScan [TS_68] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_71] - Select Operator [SEL_67] (rows=22000000 width=1014) - Output:["_col1"] - Group By Operator [GBY_66] (rows=22000000 width=1014) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0, _col1 - Group By Operator [GBY_64] (rows=44000000 width=1014) - Output:["_col0","_col1"],keys:_col7, _col1 - Select Operator [SEL_63] (rows=44000000 width=1014) - Output:["_col7","_col1"] - Merge Join Operator [MERGEJOIN_153] (rows=44000000 width=1014) - Conds:RS_60._col2=RS_61._col0(Inner),Output:["_col1","_col7"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0 - Select Operator [SEL_56] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_54] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_152] (rows=31678769 width=106) - Conds:RS_57._col0=RS_58._col0(Inner),Output:["_col1","_col2"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_149] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_returning_addr_sk is not null) - TableScan [TS_48] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_returning_addr_sk"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_53] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_150] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_51] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col19 - Merge Join Operator [MERGEJOIN_160] (rows=96800003 width=860) - Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col14","_col15","_col16","_col17","_col19","_col20"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Group By Operator [GBY_24] (rows=22000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1 - Group By Operator [GBY_22] (rows=44000000 width=1014) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col1, _col7 - Select Operator [SEL_21] (rows=44000000 width=1014) - Output:["_col1","_col7","_col3"] - Merge Join Operator [MERGEJOIN_157] (rows=44000000 width=1014) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col1","_col3","_col7"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - TableScan [TS_12] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_156] (rows=31678769 width=106) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_144] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_143] (rows=28798881 width=106) - predicate:(cr_returned_date_sk is not null and cr_returning_addr_sk is not null and cr_returning_customer_sk is not null) - TableScan [TS_6] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_returning_addr_sk","cr_return_amt_inc_tax"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_155] (rows=88000001 width=860) - Conds:RS_87._col2=RS_88._col0(Inner),Output:["_col0","_col1","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col14","_col15","_col16","_col17"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_141] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_current_addr_sk","c_salutation","c_first_name","c_last_name"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0","_col1","_col10","_col11","_col2","_col3","_col4","_col5","_col6","_col8","_col9"] - Filter Operator [FIL_142] (rows=20000000 width=1014) - predicate:((ca_state = 'IL') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_street_type","ca_suite_number","ca_city","ca_county","ca_state","ca_zip","ca_country","ca_gmt_offset","ca_location_type"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query82.q.out ql/src/test/results/clientpositive/perf/query82.q.out index 9c0042c..271a324 100644 --- ql/src/test/results/clientpositive/perf/query82.q.out +++ ql/src/test/results/clientpositive/perf/query82.q.out @@ -33,71 +33,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 4 - File Output Operator [FS_28] - Limit [LIM_27] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_26] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Group By Operator [GBY_23] (rows=633595212 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_21] (rows=1267190424 width=88) - Output:["_col0","_col1","_col2"],keys:_col2, _col3, _col4 - Merge Join Operator [MERGEJOIN_44] (rows=1267190424 width=88) - Conds:RS_16._col0=RS_17._col0(Inner),RS_17._col0=RS_18._col1(Inner),Output:["_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0"] - Filter Operator [FIL_39] (rows=575995635 width=88) - predicate:ss_item_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=25666 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_40] (rows=25666 width=1436) - predicate:((i_manufact_id) IN (437, 129, 727, 663) and i_current_price BETWEEN 30 AND 60 and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_manufact_id"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=4593600 width=15) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4176000 width=15) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=4176000 width=15) - predicate:(inv_quantity_on_hand BETWEEN 100 AND 500 and inv_item_sk is not null and inv_date_sk is not null) - TableScan [TS_6] (rows=37584000 width=15) - default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_quantity_on_hand"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_42] (rows=8116 width=1119) - predicate:(d_date BETWEEN 2002-05-30 AND 2002-07-29 00:00:00.0 and d_date_sk is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 4 diff --git ql/src/test/results/clientpositive/perf/query83.q.out ql/src/test/results/clientpositive/perf/query83.q.out index 9960bc7..0ef2bb8 100644 --- ql/src/test/results/clientpositive/perf/query83.q.out +++ ql/src/test/results/clientpositive/perf/query83.q.out @@ -131,308 +131,33 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 21 <- Map 20 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 23 <- Map 22 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE) -Reducer 26 <- Map 25 (SIMPLE_EDGE) -Reducer 28 <- Map 27 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) -Reducer 33 <- Map 32 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 36 <- Reducer 35 (SIMPLE_EDGE) -Reducer 38 <- Map 37 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 18 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE) +Vertex 16 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 19 (SORT_PARTITION_EDGE) +Vertex 17 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 21 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 23 <- Vertex 22 (SORT_PARTITION_EDGE), Vertex 26 (SORT_PARTITION_EDGE) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE) +Vertex 26 <- Vertex 25 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (SORT_PARTITION_EDGE) +Vertex 33 <- Vertex 32 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 38 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 30 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_134] - Limit [LIM_133] (rows=100 width=77) - Number of rows:100 - Select Operator [SEL_132] (rows=76653825 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_131] - Select Operator [SEL_130] (rows=76653825 width=77) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_228] (rows=76653825 width=77) - Conds:RS_126._col0=RS_127._col0(Inner),RS_126._col0=RS_128._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_127] - PartitionCols:_col0 - Group By Operator [GBY_82] (rows=34842647 width=77) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0 - Group By Operator [GBY_80] (rows=69685294 width=77) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 - Select Operator [SEL_79] (rows=69685294 width=77) - Output:["_col4","_col2"] - Merge Join Operator [MERGEJOIN_226] (rows=69685294 width=77) - Conds:RS_76._col0=RS_77._col0(Inner),Output:["_col2","_col4"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_219] (rows=63350266 width=77) - Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col1 - Select Operator [SEL_44] (rows=57591150 width=77) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_206] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_returned_date_sk is not null) - TableScan [TS_42] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_return_quantity"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_207] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_45] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_221] (rows=80353 width=1119) - Conds:RS_69._col1=RS_70._col0(Inner),Output:["_col0"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_69] - PartitionCols:_col1 - Select Operator [SEL_50] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_208] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - TableScan [TS_48] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col0 - Group By Operator [GBY_67] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Group By Operator [GBY_65] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_220] (rows=80353 width=1119) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col0"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Select Operator [SEL_53] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_209] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_51] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_62] - PartitionCols:_col0 - Group By Operator [GBY_59] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Group By Operator [GBY_57] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_56] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_210] (rows=36525 width=1119) - predicate:((d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') and d_week_seq is not null) - TableScan [TS_54] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_128] - PartitionCols:_col0 - Group By Operator [GBY_124] (rows=8711072 width=92) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_123] - PartitionCols:_col0 - Group By Operator [GBY_122] (rows=17422145 width=92) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 - Select Operator [SEL_121] (rows=17422145 width=92) - Output:["_col4","_col2"] - Merge Join Operator [MERGEJOIN_227] (rows=17422145 width=92) - Conds:RS_118._col0=RS_119._col0(Inner),Output:["_col2","_col4"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_118] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_222] (rows=15838314 width=92) - Conds:RS_115._col1=RS_116._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col1 - Select Operator [SEL_86] (rows=14398467 width=92) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_211] (rows=14398467 width=92) - predicate:(wr_item_sk is not null and wr_returned_date_sk is not null) - TableScan [TS_84] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_item_sk","wr_return_quantity"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_212] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_87] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_224] (rows=80353 width=1119) - Conds:RS_111._col1=RS_112._col0(Inner),Output:["_col0"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col1 - Select Operator [SEL_92] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_213] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - TableScan [TS_90] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col0 - Group By Operator [GBY_109] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Group By Operator [GBY_107] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_223] (rows=80353 width=1119) - Conds:RS_103._col1=RS_104._col0(Inner),Output:["_col0"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col1 - Select Operator [SEL_95] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_214] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_93] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Group By Operator [GBY_101] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_99] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_98] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_215] (rows=36525 width=1119) - predicate:((d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') and d_week_seq is not null) - TableScan [TS_96] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0 - Group By Operator [GBY_40] (rows=17423323 width=106) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Group By Operator [GBY_38] (rows=34846646 width=106) - Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 - Select Operator [SEL_37] (rows=34846646 width=106) - Output:["_col4","_col2"] - Merge Join Operator [MERGEJOIN_225] (rows=34846646 width=106) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col2","_col4"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_216] (rows=31678769 width=106) - Conds:RS_31._col1=RS_32._col0(Inner),Output:["_col0","_col2","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=28798881 width=106) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_201] (rows=28798881 width=106) - predicate:(cr_item_sk is not null and cr_returned_date_sk is not null) - TableScan [TS_0] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_item_sk","cr_return_quantity"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_202] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_218] (rows=80353 width=1119) - Conds:RS_27._col1=RS_28._col0(Inner),Output:["_col0"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_203] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Group By Operator [GBY_25] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_217] (rows=80353 width=1119) - Conds:RS_19._col1=RS_20._col0(Inner),Output:["_col0"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_204] (rows=73049 width=1119) - predicate:(d_week_seq is not null and d_date is not null) - TableScan [TS_9] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=18262 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=36525 width=1119) - Output:["_col0"],keys:d_week_seq - Select Operator [SEL_14] (rows=36525 width=1119) - Output:["d_week_seq"] - Filter Operator [FIL_205] (rows=36525 width=1119) - predicate:((d_date) IN ('1998-01-02', '1998-10-15', '1998-11-10') and d_week_seq is not null) - TableScan [TS_12] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query84.q.out ql/src/test/results/clientpositive/perf/query84.q.out index cf68729..8ccd524 100644 --- ql/src/test/results/clientpositive/perf/query84.q.out +++ ql/src/test/results/clientpositive/perf/query84.q.out @@ -5,95 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 11 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 7 <- Reducer 10 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Vertex 10 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 3 - File Output Operator [FS_37] - Limit [LIM_36] (rows=100 width=860) - Number of rows:100 - Select Operator [SEL_35] (rows=212960011 width=860) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=212960011 width=860) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_67] (rows=212960011 width=860) - Conds:RS_29._col0=RS_30._col0(Inner),RS_30._col0=RS_31._col1(Inner),Output:["_col2","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0"] - Filter Operator [FIL_58] (rows=57591150 width=77) - predicate:sr_cdemo_sk is not null - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_cdemo_sk"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1861800 width=385) - Output:["_col0"] - Filter Operator [FIL_59] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - TableScan [TS_3] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_66] (rows=96800003 width=860) - Conds:RS_25._col2=RS_26._col0(Inner),Output:["_col0","_col1","_col4","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_65] (rows=7920 width=107) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col0"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=2 width=12) - Output:["_col0"] - Filter Operator [FIL_63] (rows=2 width=12) - predicate:((ib_lower_bound >= 32287) and (ib_upper_bound <= 82287) and ib_income_band_sk is not null) - TableScan [TS_15] (rows=20 width=12) - default@income_band,income_band,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk","ib_lower_bound","ib_upper_bound"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_62] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_12] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_64] (rows=88000001 width=860) - Conds:RS_22._col3=RS_23._col0(Inner),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col3 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_60] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null) - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_id","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_name","c_last_name"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_61] (rows=20000000 width=1014) - predicate:((ca_city = 'Hopewell') and ca_address_sk is not null) - TableScan [TS_9] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_city"] + Vertex 3 diff --git ql/src/test/results/clientpositive/perf/query85.q.out ql/src/test/results/clientpositive/perf/query85.q.out index e7f7348..45918ea 100644 --- ql/src/test/results/clientpositive/perf/query85.q.out +++ ql/src/test/results/clientpositive/perf/query85.q.out @@ -5,153 +5,19 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Map 17 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 7 <- Map 6 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 8 <- Map 16 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_59] - Limit [LIM_58] (rows=100 width=1014) - Number of rows:100 - Select Operator [SEL_57] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Select Operator [SEL_55] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_54] (rows=1220082 width=1014) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Group By Operator [GBY_52] (rows=2440165 width=1014) - Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col6)","avg(_col16)","avg(_col15)"],keys:_col28 - Select Operator [SEL_51] (rows=2440165 width=1014) - Output:["_col28","_col6","_col16","_col15"] - Merge Join Operator [MERGEJOIN_109] (rows=2440165 width=1014) - Conds:RS_48._col13=RS_49._col0(Inner),Output:["_col6","_col15","_col16","_col28"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=72 width=200) - Output:["_col0","_col1"] - Filter Operator [FIL_102] (rows=72 width=200) - predicate:r_reason_sk is not null - TableScan [TS_42] (rows=72 width=200) - default@reason,reason,Tbl:COMPLETE,Col:NONE,Output:["r_reason_sk","r_reason_desc"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col13 - Merge Join Operator [MERGEJOIN_108] (rows=2218332 width=1014) - Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col6","_col13","_col15","_col16"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_95] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - TableScan [TS_0] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=2016666 width=1014) - Output:["_col0","_col11","_col13","_col14","_col4"] - Filter Operator [FIL_40] (rows=2016666 width=1014) - predicate:(((_col23) IN ('KY', 'GA', 'NM') and _col6 BETWEEN 100 AND 200) or ((_col23) IN ('MT', 'OR', 'IN') and _col6 BETWEEN 150 AND 300) or ((_col23) IN ('WI', 'MO', 'WV') and _col6 BETWEEN 50 AND 250)) - Select Operator [SEL_39] (rows=12100000 width=1014) - Output:["_col0","_col4","_col6","_col11","_col13","_col14","_col23"] - Merge Join Operator [MERGEJOIN_107] (rows=12100000 width=1014) - Conds:RS_36._col13, _col20, _col21=RS_37._col0, _col1, _col2(Inner),Output:["_col1","_col3","_col7","_col9","_col14","_col16","_col17"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_32] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_101] (rows=1861800 width=385) - predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) - TableScan [TS_30] (rows=1861800 width=385) - default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col13, _col20, _col21 - Merge Join Operator [MERGEJOIN_106] (rows=11000000 width=1014) - Conds:RS_33._col0=RS_34._col9(Inner),Output:["_col1","_col3","_col7","_col9","_col13","_col14","_col16","_col17","_col20","_col21"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=10000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=10000000 width=1014) - predicate:((ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and (ca_country = 'United States') and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state","ca_country"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col9 - Select Operator [SEL_29] (rows=1774698 width=135) - Output:["_col0","_col10","_col11","_col13","_col14","_col17","_col18","_col4","_col6","_col9"] - Filter Operator [FIL_28] (rows=1774698 width=135) - predicate:(((_col17 = 'M') and (_col18 = '4 yr Degree') and _col5 BETWEEN 100 AND 150) or ((_col17 = 'D') and (_col18 = 'Primary') and _col5 BETWEEN 50 AND 100) or ((_col17 = 'U') and (_col18 = 'Advanced Degree') and _col5 BETWEEN 150 AND 200)) - Select Operator [SEL_27] (rows=21296393 width=135) - Output:["_col0","_col4","_col5","_col6","_col9","_col10","_col11","_col13","_col14","_col17","_col18"] - Merge Join Operator [MERGEJOIN_105] (rows=21296393 width=135) - Conds:RS_24._col9=RS_25._col0(Inner),Output:["_col1","_col5","_col6","_col7","_col10","_col11","_col12","_col14","_col15","_col17","_col18"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_100] (rows=1861800 width=385) - predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) - TableScan [TS_15] (rows=1861800 width=385) - default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col9 - Merge Join Operator [MERGEJOIN_104] (rows=19360357 width=135) - Conds:RS_21._col2, _col4=RS_22._col0, _col5(Inner),Output:["_col1","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col14","_col15"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0, _col5 - Select Operator [SEL_14] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_99] (rows=14398467 width=92) - predicate:(wr_item_sk is not null and wr_order_number is not null and wr_refunded_cdemo_sk is not null and wr_returning_cdemo_sk is not null and wr_refunded_addr_sk is not null and wr_reason_sk is not null) - TableScan [TS_12] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_refunded_cdemo_sk","wr_refunded_addr_sk","wr_returning_cdemo_sk","wr_reason_sk","wr_order_number","wr_fee","wr_refunded_cash"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2, _col4 - Merge Join Operator [MERGEJOIN_103] (rows=17600325 width=135) - Conds:RS_18._col0=RS_19._col2(Inner),Output:["_col1","_col2","_col4","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col2 - Select Operator [SEL_11] (rows=16000296 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_98] (rows=16000296 width=135) - predicate:((ws_sales_price BETWEEN 100 AND 150 or ws_sales_price BETWEEN 50 AND 100 or ws_sales_price BETWEEN 150 AND 200) and (ws_net_profit BETWEEN 100 AND 200 or ws_net_profit BETWEEN 150 AND 300 or ws_net_profit BETWEEN 50 AND 250) and ws_order_number is not null and ws_item_sk is not null and ws_web_page_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_9] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_web_page_sk","ws_order_number","ws_quantity","ws_sales_price","ws_net_profit"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=4602 width=585) - Output:["_col0"] - Filter Operator [FIL_97] (rows=4602 width=585) - predicate:wp_web_page_sk is not null - TableScan [TS_6] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query86.q.out ql/src/test/results/clientpositive/perf/query86.q.out index 6377c43..8e10e58 100644 --- ql/src/test/results/clientpositive/perf/query86.q.out +++ ql/src/test/results/clientpositive/perf/query86.q.out @@ -51,76 +51,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_29] - Limit [LIM_28] (rows=100 width=135) - Number of rows:100 - Select Operator [SEL_27] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - Select Operator [SEL_24] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - PTF Operator [PTF_23] (rows=261364852 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END"}] - Select Operator [SEL_22] (rows=261364852 width=135) - Output:["_col0","_col1","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((UDFToInteger(grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END - Select Operator [SEL_20] (rows=261364852 width=135) - Output:["_col0","_col1","_col4","_col5"] - Group By Operator [GBY_19] (rows=261364852 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_17] (rows=522729705 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col2)"],keys:_col0, _col1, 0 - Select Operator [SEL_15] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_39] (rows=174243235 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_37] (rows=462000 width=1436) - predicate:i_item_sk is not null - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_38] (rows=158402938 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_35] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_item_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_net_paid"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_36] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query87.q.out ql/src/test/results/clientpositive/perf/query87.q.out index 09f1539..8d752e4 100644 --- ql/src/test/results/clientpositive/perf/query87.q.out +++ ql/src/test/results/clientpositive/perf/query87.q.out @@ -43,217 +43,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 20 <- Map 23 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Reducer 20 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) -Reducer 8 <- Union 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 5 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 10 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 21 <- Union 7 (CONTAINS), Vertex 20 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 - File Output Operator [FS_104] - Group By Operator [GBY_102] (rows=1 width=24) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 8 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_101] - Group By Operator [GBY_100] (rows=1 width=24) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_98] (rows=4537552 width=129) - Filter Operator [FIL_97] (rows=4537552 width=129) - predicate:((_col3 > 0) and ((_col3 * 2) = _col4)) - Select Operator [SEL_129] (rows=27225312 width=129) - Output:["_col3","_col4"] - Group By Operator [GBY_96] (rows=27225312 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 7 [SIMPLE_EDGE] - <-Reducer 21 [CONTAINS] - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_94] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_92] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_90] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_89] (rows=43560808 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_83] (rows=87121617 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_81] (rows=174243235 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_80] (rows=174243235 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_135] (rows=174243235 width=135) - Conds:RS_77._col1=RS_78._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_78] - PartitionCols:_col0 - Select Operator [SEL_73] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_128] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_71] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_77] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_134] (rows=158402938 width=135) - Conds:RS_74._col0=RS_75._col0(Inner),Output:["_col1","_col3"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Select Operator [SEL_67] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_customer_sk is not null) - TableScan [TS_65] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Select Operator [SEL_70] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_127] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_68] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 6 [CONTAINS] - Reduce Output Operator [RS_95] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_94] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_92] (rows=54450625 width=129) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_64] (rows=10889817 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_63] (rows=10889817 width=103) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(2)"],keys:_col0, _col1, _col2 - Select Operator [SEL_59] (rows=21779634 width=103) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_58] (rows=21779634 width=103) - predicate:((_col3 > 0) and ((_col3 * 2) = _col4)) - Group By Operator [GBY_57] (rows=130677808 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_55] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_53] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_51] (rows=87116929 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_50] (rows=87116929 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(1)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_44] (rows=174233858 width=135) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_42] (rows=348467716 width=135) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_41] (rows=348467716 width=135) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_133] (rows=348467716 width=135) - Conds:RS_38._col1=RS_39._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_125] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_32] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_132] (rows=316788826 width=135) - Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col1","_col3"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_123] (rows=287989836 width=135) - predicate:(cs_sold_date_sk is not null and cs_bill_customer_sk is not null) - TableScan [TS_26] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_124] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_29] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_56] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_55] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","sum(_col4)"],keys:_col0, _col1, _col2 - Select Operator [SEL_53] (rows=261355616 width=103) - Output:["_col0","_col1","_col2","_col3","_col4"] - Select Operator [SEL_25] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Group By Operator [GBY_24] (rows=174238687 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(2)"],keys:_col0, _col1, _col2 - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"],keys:_col7, _col6, _col3 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col7","_col6","_col3"] - Merge Join Operator [MERGEJOIN_131] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col3","_col6","_col7"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_122] (rows=80000000 width=860) - predicate:c_customer_sk is not null - TableScan [TS_6] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_130] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_120] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_customer_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_121] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] + Vertex 9 diff --git ql/src/test/results/clientpositive/perf/query88.q.out ql/src/test/results/clientpositive/perf/query88.q.out index 5072997..b18808f 100644 --- ql/src/test/results/clientpositive/perf/query88.q.out +++ ql/src/test/results/clientpositive/perf/query88.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[347][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[347][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain select * from @@ -186,496 +186,43 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) -Reducer 12 <- Map 16 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Map 24 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 25 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (CUSTOM_SIMPLE_EDGE) -Reducer 27 <- Map 26 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 28 <- Map 32 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Map 33 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (CUSTOM_SIMPLE_EDGE) -Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 39 (SIMPLE_EDGE) -Reducer 36 <- Map 40 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE) -Reducer 37 <- Map 41 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 47 (SIMPLE_EDGE) -Reducer 44 <- Map 48 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 45 <- Map 49 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE) -Reducer 46 <- Reducer 45 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 51 <- Map 50 (SIMPLE_EDGE), Map 55 (SIMPLE_EDGE) -Reducer 52 <- Map 56 (SIMPLE_EDGE), Reducer 51 (SIMPLE_EDGE) -Reducer 53 <- Map 57 (SIMPLE_EDGE), Reducer 52 (SIMPLE_EDGE) -Reducer 54 <- Reducer 53 (CUSTOM_SIMPLE_EDGE) -Reducer 59 <- Map 58 (SIMPLE_EDGE), Map 63 (SIMPLE_EDGE) -Reducer 6 <- Reducer 14 (CUSTOM_SIMPLE_EDGE), Reducer 22 (CUSTOM_SIMPLE_EDGE), Reducer 30 (CUSTOM_SIMPLE_EDGE), Reducer 38 (CUSTOM_SIMPLE_EDGE), Reducer 46 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 54 (CUSTOM_SIMPLE_EDGE), Reducer 62 (CUSTOM_SIMPLE_EDGE) -Reducer 60 <- Map 64 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE) -Reducer 61 <- Map 65 (SIMPLE_EDGE), Reducer 60 (SIMPLE_EDGE) -Reducer 62 <- Reducer 61 (CUSTOM_SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (PARTITION_EDGE) +Vertex 19 <- Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 20 <- Vertex 19 (SORT_PARTITION_EDGE), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 21 <- Vertex 20 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 22 <- Vertex 21 (PARTITION_EDGE) +Vertex 27 <- Vertex 26 (SORT_PARTITION_EDGE), Vertex 31 (SORT_PARTITION_EDGE) +Vertex 28 <- Vertex 27 (SORT_PARTITION_EDGE), Vertex 32 (SORT_PARTITION_EDGE) +Vertex 29 <- Vertex 28 (SORT_PARTITION_EDGE), Vertex 33 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 30 <- Vertex 29 (PARTITION_EDGE) +Vertex 35 <- Vertex 34 (SORT_PARTITION_EDGE), Vertex 39 (SORT_PARTITION_EDGE) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 40 (SORT_PARTITION_EDGE) +Vertex 37 <- Vertex 36 (SORT_PARTITION_EDGE), Vertex 41 (SORT_PARTITION_EDGE) +Vertex 38 <- Vertex 37 (PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 43 <- Vertex 42 (SORT_PARTITION_EDGE), Vertex 47 (SORT_PARTITION_EDGE) +Vertex 44 <- Vertex 43 (SORT_PARTITION_EDGE), Vertex 48 (SORT_PARTITION_EDGE) +Vertex 45 <- Vertex 44 (SORT_PARTITION_EDGE), Vertex 49 (SORT_PARTITION_EDGE) +Vertex 46 <- Vertex 45 (PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 51 <- Vertex 50 (SORT_PARTITION_EDGE), Vertex 55 (SORT_PARTITION_EDGE) +Vertex 52 <- Vertex 51 (SORT_PARTITION_EDGE), Vertex 56 (SORT_PARTITION_EDGE) +Vertex 53 <- Vertex 52 (SORT_PARTITION_EDGE), Vertex 57 (SORT_PARTITION_EDGE) +Vertex 54 <- Vertex 53 (PARTITION_EDGE) +Vertex 59 <- Vertex 58 (SORT_PARTITION_EDGE), Vertex 63 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 14 (PARTITION_EDGE), Vertex 22 (PARTITION_EDGE), Vertex 30 (PARTITION_EDGE), Vertex 38 (PARTITION_EDGE), Vertex 46 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE), Vertex 54 (PARTITION_EDGE), Vertex 62 (PARTITION_EDGE) +Vertex 60 <- Vertex 59 (SORT_PARTITION_EDGE), Vertex 64 (SORT_PARTITION_EDGE) +Vertex 61 <- Vertex 60 (SORT_PARTITION_EDGE), Vertex 65 (SORT_PARTITION_EDGE) +Vertex 62 <- Vertex 61 (PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_218] - Select Operator [SEL_217] (rows=1 width=65) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_347] (rows=1 width=65) - Conds:(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 14 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_209] - Group By Operator [GBY_50] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 13 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_49] - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_328] (rows=766650239 width=88) - Conds:RS_44._col1=RS_45._col0(Inner) - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_298] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_35] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_327] (rows=696954748 width=88) - Conds:RS_41._col2=RS_42._col0(Inner),Output:["_col1"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_297] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_32] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_326] (rows=633595212 width=88) - Conds:RS_38._col0=RS_39._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_28] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_295] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_26] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_296] (rows=14400 width=471) - predicate:((t_hour = 12) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_29] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 22 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_210] - Group By Operator [GBY_76] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_75] - Group By Operator [GBY_74] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_331] (rows=766650239 width=88) - Conds:RS_70._col1=RS_71._col0(Inner) - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Select Operator [SEL_63] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_302] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_61] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_70] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_330] (rows=696954748 width=88) - Conds:RS_67._col2=RS_68._col0(Inner),Output:["_col1"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_68] - PartitionCols:_col0 - Select Operator [SEL_60] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_301] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_58] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_329] (rows=633595212 width=88) - Conds:RS_64._col0=RS_65._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col0 - Select Operator [SEL_54] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_299] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_52] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0 - Select Operator [SEL_57] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_300] (rows=14400 width=471) - predicate:((t_hour = 11) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_55] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 30 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_211] - Group By Operator [GBY_102] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 29 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_101] - Group By Operator [GBY_100] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_334] (rows=766650239 width=88) - Conds:RS_96._col1=RS_97._col0(Inner) - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_306] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_87] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_333] (rows=696954748 width=88) - Conds:RS_93._col2=RS_94._col0(Inner),Output:["_col1"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_86] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_305] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_84] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_332] (rows=633595212 width=88) - Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col2"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_90] - PartitionCols:_col0 - Select Operator [SEL_80] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_303] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_78] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_91] - PartitionCols:_col0 - Select Operator [SEL_83] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_304] (rows=14400 width=471) - predicate:((t_hour = 11) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_81] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 38 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_212] - Group By Operator [GBY_128] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 37 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_127] - Group By Operator [GBY_126] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_337] (rows=766650239 width=88) - Conds:RS_122._col1=RS_123._col0(Inner) - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_123] - PartitionCols:_col0 - Select Operator [SEL_115] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_310] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_113] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_336] (rows=696954748 width=88) - Conds:RS_119._col2=RS_120._col0(Inner),Output:["_col1"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_120] - PartitionCols:_col0 - Select Operator [SEL_112] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_309] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_110] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 35 [SIMPLE_EDGE] - SHUFFLE [RS_119] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_335] (rows=633595212 width=88) - Conds:RS_116._col0=RS_117._col0(Inner),Output:["_col1","_col2"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_116] - PartitionCols:_col0 - Select Operator [SEL_106] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_307] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_104] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_117] - PartitionCols:_col0 - Select Operator [SEL_109] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_308] (rows=14400 width=471) - predicate:((t_hour = 10) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_107] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 46 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_213] - Group By Operator [GBY_154] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 45 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_153] - Group By Operator [GBY_152] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_340] (rows=766650239 width=88) - Conds:RS_148._col1=RS_149._col0(Inner) - <-Map 49 [SIMPLE_EDGE] - SHUFFLE [RS_149] - PartitionCols:_col0 - Select Operator [SEL_141] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_314] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_139] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_148] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_339] (rows=696954748 width=88) - Conds:RS_145._col2=RS_146._col0(Inner),Output:["_col1"] - <-Map 48 [SIMPLE_EDGE] - SHUFFLE [RS_146] - PartitionCols:_col0 - Select Operator [SEL_138] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_313] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_136] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_145] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_338] (rows=633595212 width=88) - Conds:RS_142._col0=RS_143._col0(Inner),Output:["_col1","_col2"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_142] - PartitionCols:_col0 - Select Operator [SEL_132] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_311] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_130] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col0 - Select Operator [SEL_135] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_312] (rows=14400 width=471) - predicate:((t_hour = 10) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_133] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_208] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_325] (rows=766650239 width=88) - Conds:RS_18._col1=RS_19._col0(Inner) - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_294] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_324] (rows=696954748 width=88) - Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_293] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_323] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_291] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_292] (rows=14400 width=471) - predicate:((t_hour = 8) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 54 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_214] - Group By Operator [GBY_180] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 53 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_179] - Group By Operator [GBY_178] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_343] (rows=766650239 width=88) - Conds:RS_174._col1=RS_175._col0(Inner) - <-Map 57 [SIMPLE_EDGE] - SHUFFLE [RS_175] - PartitionCols:_col0 - Select Operator [SEL_167] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_318] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_165] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 52 [SIMPLE_EDGE] - SHUFFLE [RS_174] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_342] (rows=696954748 width=88) - Conds:RS_171._col2=RS_172._col0(Inner),Output:["_col1"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_172] - PartitionCols:_col0 - Select Operator [SEL_164] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_317] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_162] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 51 [SIMPLE_EDGE] - SHUFFLE [RS_171] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_341] (rows=633595212 width=88) - Conds:RS_168._col0=RS_169._col0(Inner),Output:["_col1","_col2"] - <-Map 50 [SIMPLE_EDGE] - SHUFFLE [RS_168] - PartitionCols:_col0 - Select Operator [SEL_158] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_315] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_156] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_169] - PartitionCols:_col0 - Select Operator [SEL_161] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_316] (rows=14400 width=471) - predicate:((t_hour = 9) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_159] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] - <-Reducer 62 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_215] - Group By Operator [GBY_206] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 61 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_205] - Group By Operator [GBY_204] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_346] (rows=766650239 width=88) - Conds:RS_200._col1=RS_201._col0(Inner) - <-Map 65 [SIMPLE_EDGE] - SHUFFLE [RS_201] - PartitionCols:_col0 - Select Operator [SEL_193] (rows=6000 width=107) - Output:["_col0"] - Filter Operator [FIL_322] (rows=6000 width=107) - predicate:(((hd_dep_count = 3) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - TableScan [TS_191] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] - <-Reducer 60 [SIMPLE_EDGE] - SHUFFLE [RS_200] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_345] (rows=696954748 width=88) - Conds:RS_197._col2=RS_198._col0(Inner),Output:["_col1"] - <-Map 64 [SIMPLE_EDGE] - SHUFFLE [RS_198] - PartitionCols:_col0 - Select Operator [SEL_190] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_321] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_188] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 59 [SIMPLE_EDGE] - SHUFFLE [RS_197] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_344] (rows=633595212 width=88) - Conds:RS_194._col0=RS_195._col0(Inner),Output:["_col1","_col2"] - <-Map 58 [SIMPLE_EDGE] - SHUFFLE [RS_194] - PartitionCols:_col0 - Select Operator [SEL_184] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_319] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_182] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 63 [SIMPLE_EDGE] - SHUFFLE [RS_195] - PartitionCols:_col0 - Select Operator [SEL_187] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_320] (rows=14400 width=471) - predicate:((t_hour = 9) and (t_minute < 30) and t_time_sk is not null) - TableScan [TS_185] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query89.q.out ql/src/test/results/clientpositive/perf/query89.q.out index 7bc8700..4ec3f63 100644 --- ql/src/test/results/clientpositive/perf/query89.q.out +++ ql/src/test/results/clientpositive/perf/query89.q.out @@ -55,95 +55,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_36] - Limit [LIM_35] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_34] (rows=191662559 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_33] - Select Operator [SEL_30] (rows=191662559 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_46] (rows=191662559 width=88) - predicate:CASE WHEN ((avg_window_0 <> 0)) THEN (((abs((_col6 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END - Select Operator [SEL_29] (rows=383325119 width=88) - Output:["avg_window_0","_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_28] (rows=383325119 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col2, _col3, _col4"}] - Select Operator [SEL_27] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col2, _col3, _col4 - Select Operator [SEL_25] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_24] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_22] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col7, _col10, _col12, _col13 - Select Operator [SEL_21] (rows=766650239 width=88) - Output:["_col5","_col6","_col7","_col10","_col12","_col13","_col3"] - Merge Join Operator [MERGEJOIN_53] (rows=766650239 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col7","_col10","_col12","_col13"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_50] (rows=1704 width=1910) - predicate:s_store_sk is not null - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_52] (rows=696954748 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col7","_col10"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36525 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_49] (rows=36525 width=1119) - predicate:((d_year) IN (2000) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_51] (rows=633595212 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col0","_col2","_col3","_col5","_col6","_col7"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_47] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_48] (rows=231000 width=1436) - predicate:(((i_class) IN ('wallpaper', 'parenting', 'musical') or (i_class) IN ('womens', 'birdal', 'pants')) and ((i_category) IN ('Home', 'Books', 'Electronics') or (i_category) IN ('Shoes', 'Jewelry', 'Men')) and (((i_category) IN ('Home', 'Books', 'Electronics') and (i_class) IN ('wallpaper', 'parenting', 'musical')) or ((i_category) IN ('Shoes', 'Jewelry', 'Men') and (i_class) IN ('womens', 'birdal', 'pants'))) and i_item_sk is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query90.q.out ql/src/test/results/clientpositive/perf/query90.q.out index 5ae9fe5..d0e4e46 100644 --- ql/src/test/results/clientpositive/perf/query90.q.out +++ ql/src/test/results/clientpositive/perf/query90.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_0, $hdt$_1]] in Stage 'Vertex 6' is a cross product PREHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 PREHOOK: type: QUERY POSTHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 @@ -6,143 +6,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) -Reducer 13 <- Map 17 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 18 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Reducer 14 (CUSTOM_SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 13 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 15 <- Vertex 14 (PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 6 <- Vertex 15 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_59] - Limit [LIM_58] (rows=1 width=17) - Number of rows:100 - Select Operator [SEL_57] (rows=1 width=17) - Output:["_col0"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_56] - Select Operator [SEL_55] (rows=1 width=17) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_92] (rows=1 width=17) - Conds:(Inner),Output:["_col0","_col1"] - <-Reducer 15 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_53] - Group By Operator [GBY_50] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 14 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_49] - Group By Operator [GBY_48] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_91] (rows=191667562 width=135) - Conds:RS_44._col1=RS_45._col0(Inner) - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_85] (rows=3600 width=107) - predicate:((hd_dep_count = 8) and hd_demo_sk is not null) - TableScan [TS_35] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_90] (rows=174243235 width=135) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_84] (rows=9600 width=471) - predicate:(t_hour BETWEEN 14 AND 15 and t_time_sk is not null) - TableScan [TS_32] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_89] (rows=158402938 width=135) - Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col0","_col1"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Select Operator [SEL_28] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=144002668 width=135) - predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) - TableScan [TS_26] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0 - Select Operator [SEL_31] (rows=511 width=585) - Output:["_col0"] - Filter Operator [FIL_83] (rows=511 width=585) - predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) - TableScan [TS_29] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] - <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_88] (rows=191667562 width=135) - Conds:RS_18._col1=RS_19._col0(Inner) - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_81] (rows=3600 width=107) - predicate:((hd_dep_count = 8) and hd_demo_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_87] (rows=174243235 width=135) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_80] (rows=9600 width=471) - predicate:(t_hour BETWEEN 6 AND 7 and t_time_sk is not null) - TableScan [TS_6] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_86] (rows=158402938 width=135) - Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=144002668 width=135) - predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=511 width=585) - Output:["_col0"] - Filter Operator [FIL_79] (rows=511 width=585) - predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) - TableScan [TS_3] (rows=4602 width=585) - default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query91.q.out ql/src/test/results/clientpositive/perf/query91.q.out index 1538e8b..fda2585 100644 --- ql/src/test/results/clientpositive/perf/query91.q.out +++ ql/src/test/results/clientpositive/perf/query91.q.out @@ -5,126 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 11 <- Map 14 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 15 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 10 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_47] - Select Operator [SEL_46] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_45] - Select Operator [SEL_44] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3"] - Group By Operator [GBY_43] (rows=58564004 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_41] (rows=117128008 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col8, _col9, _col10, _col18, _col19 - Select Operator [SEL_40] (rows=117128008 width=860) - Output:["_col8","_col9","_col10","_col18","_col19","_col3"] - Merge Join Operator [MERGEJOIN_81] (rows=117128008 width=860) - Conds:RS_37._col1=RS_38._col2(Inner),Output:["_col3","_col8","_col9","_col10","_col18","_col19"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col2 - Select Operator [SEL_30] (rows=106480005 width=860) - Output:["_col2","_col7","_col8"] - Merge Join Operator [MERGEJOIN_80] (rows=106480005 width=860) - Conds:RS_27._col2=RS_28._col0(Inner),Output:["_col0","_col5","_col6"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_75] (rows=3600 width=107) - predicate:((hd_buy_potential like '0-500%') and hd_demo_sk is not null) - TableScan [TS_18] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_buy_potential"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_79] (rows=96800003 width=860) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col0","_col2","_col5","_col6"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_74] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -7) and ca_address_sk is not null) - TableScan [TS_15] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_78] (rows=88000001 width=860) - Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col0","_col2","_col3","_col5","_col6"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=930900 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_73] (rows=930900 width=385) - predicate:(((cd_education_status = 'Unknown') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'W')) and (((cd_marital_status = 'M') and (cd_education_status = 'Unknown')) or ((cd_marital_status = 'W') and (cd_education_status = 'Advanced Degree'))) and cd_demo_sk is not null) - TableScan [TS_12] (rows=1861800 width=385) - default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_72] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null) - TableScan [TS_9] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_77] (rows=34846646 width=106) - Conds:RS_34._col2=RS_35._col0(Inner),Output:["_col1","_col3","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=60 width=2045) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_71] (rows=60 width=2045) - predicate:cc_call_center_sk is not null - TableScan [TS_6] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_call_center_id","cc_name","cc_manager"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_76] (rows=31678769 width=106) - Conds:RS_31._col0=RS_32._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_69] (rows=28798881 width=106) - predicate:(cr_call_center_sk is not null and cr_returned_date_sk is not null and cr_returning_customer_sk is not null) - TableScan [TS_0] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_call_center_sk","cr_net_loss"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_70] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 11) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query92.q.out ql/src/test/results/clientpositive/perf/query92.q.out index ca1f417..bf75d3e 100644 --- ql/src/test/results/clientpositive/perf/query92.q.out +++ ql/src/test/results/clientpositive/perf/query92.q.out @@ -5,89 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_37] - Group By Operator [GBY_35] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_34] - Group By Operator [GBY_33] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col0)","sum(_col1)","sum(_col2)"] - Select Operator [SEL_31] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_48] (rows=348477374 width=88) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Outer),Output:["_col0","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_46] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_42] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,ss,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_43] (rows=8116 width=1119) - predicate:((d_month_seq >= 1206) and (d_month_seq <= 1217) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Group By Operator [GBY_26] (rows=158394413 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=316788826 width=135) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_47] (rows=316788826 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_45] (rows=8116 width=1119) - predicate:((d_month_seq >= 1206) and (d_month_seq <= 1217) and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_44] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_14] (rows=287989836 width=135) - default@catalog_sales,cs,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query93.q.out ql/src/test/results/clientpositive/perf/query93.q.out index f28ba41..57ea1e0 100644 --- ql/src/test/results/clientpositive/perf/query93.q.out +++ ql/src/test/results/clientpositive/perf/query93.q.out @@ -5,64 +5,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) - Number of rows:100 - Select Operator [SEL_22] (rows=316797606 width=88) - Output:["_col0","_col1"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Group By Operator [GBY_19] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Group By Operator [GBY_17] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_15] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_34] (rows=633595212 width=88) - Conds:RS_12._col0, _col2=RS_13._col0, _col2(Inner),Output:["_col3","_col7","_col9","_col10"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0, _col2 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_32] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null) - TableScan [TS_6] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk","ss_customer_sk","ss_ticket_number","ss_quantity","ss_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0, _col2 - Merge Join Operator [MERGEJOIN_33] (rows=63350266 width=77) - Conds:RS_9._col1=RS_10._col0(Inner),Output:["_col0","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_30] (rows=57591150 width=77) - predicate:(sr_reason_sk is not null and sr_item_sk is not null and sr_ticket_number is not null) - TableScan [TS_0] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_reason_sk","sr_ticket_number","sr_return_quantity"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36 width=200) - Output:["_col0"] - Filter Operator [FIL_31] (rows=36 width=200) - predicate:((r_reason_desc = 'Did not like the warranty') and r_reason_sk is not null) - TableScan [TS_3] (rows=72 width=200) - default@reason,reason,Tbl:COMPLETE,Col:NONE,Output:["r_reason_sk","r_reason_desc"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query94.q.out ql/src/test/results/clientpositive/perf/query94.q.out index 836b16b..43b7f98 100644 --- ql/src/test/results/clientpositive/perf/query94.q.out +++ ql/src/test/results/clientpositive/perf/query94.q.out @@ -5,126 +5,17 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 14 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 9 <- Map 10 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 12 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_51] - Limit [LIM_50] (rows=1 width=344) - Number of rows:100 - Group By Operator [GBY_48] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_47] - Group By Operator [GBY_46] (rows=127554770 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col3)","sum(_col4)","sum(_col5)"],keys:_col3 - Select Operator [SEL_45] (rows=127554770 width=135) - Output:["_col3","_col4","_col5"] - Filter Operator [FIL_44] (rows=127554770 width=135) - predicate:_col12 is null - Merge Join Operator [MERGEJOIN_85] (rows=255109540 width=135) - Conds:RS_40._col3=RS_41._col0(Left Outer),Output:["_col3","_col4","_col5","_col12"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_25] (rows=14398467 width=92) - Output:["_col0"] - Filter Operator [FIL_79] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_23] (rows=14398467 width=92) - default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_84] (rows=231917759 width=135) - Conds:RS_37._col2=RS_38._col0(Inner),Output:["_col3","_col4","_col5"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Select Operator [SEL_22] (rows=42 width=1850) - Output:["_col0"] - Filter Operator [FIL_78] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_20] (rows=84 width=1850) - default@web_site,s,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_83] (rows=210834322 width=135) - Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_77] (rows=20000000 width=1014) - predicate:((ca_state = 'TX') and ca_address_sk is not null) - TableScan [TS_17] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_82] (rows=191667562 width=135) - Conds:RS_31._col0=RS_32._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_76] (rows=8116 width=1119) - predicate:(d_date BETWEEN '1999-05-01' AND '1999-07-01' and d_date_sk is not null) - TableScan [TS_14] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_31] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_81] (rows=174243235 width=135) - Conds:RS_28._col3=RS_29._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_73] (rows=144002668 width=135) - predicate:(ws_ship_addr_sk is not null and ws_web_site_sk is not null and ws_ship_date_sk is not null and ws_order_number is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Group By Operator [GBY_27] (rows=158402938 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_13] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_12] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_80] (rows=158402938 width=135) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_6] (rows=144002668 width=135) - default@web_sales,ws3,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_3] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + Vertex 7 diff --git ql/src/test/results/clientpositive/perf/query95.q.out ql/src/test/results/clientpositive/perf/query95.q.out index 9b0d1b2..7812cc2 100644 --- ql/src/test/results/clientpositive/perf/query95.q.out +++ ql/src/test/results/clientpositive/perf/query95.q.out @@ -5,152 +5,18 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) -Reducer 12 <- Map 14 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) -Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 13 (SORT_PARTITION_EDGE) +Vertex 12 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 14 (SORT_PARTITION_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 15 (SORT_PARTITION_EDGE), Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 16 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) +Vertex 8 <- Vertex 7 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_63] - Group By Operator [GBY_61] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_60] - Group By Operator [GBY_59] (rows=510219083 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col3)","sum(_col4)","sum(_col5)"],keys:_col3 - Merge Join Operator [MERGEJOIN_122] (rows=510219083 width=135) - Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col3","_col4","_col5"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_40] (rows=42 width=1850) - Output:["_col0"] - Filter Operator [FIL_115] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_38] (rows=84 width=1850) - default@web_site,s,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_121] (rows=463835520 width=135) - Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Select Operator [SEL_37] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_114] (rows=20000000 width=1014) - predicate:((ca_state = 'GA') and ca_address_sk is not null) - TableScan [TS_35] (rows=40000000 width=1014) - default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_120] (rows=421668646 width=135) - Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_50] - PartitionCols:_col0 - Select Operator [SEL_34] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_113] (rows=8116 width=1119) - predicate:(d_date BETWEEN '2002-05-01' AND '2002-06-30' and d_date_sk is not null) - TableScan [TS_32] (rows=73049 width=1119) - default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_119] (rows=383335125 width=135) - Conds:RS_45._col3=RS_46._col0(Left Semi),RS_45._col3=RS_47._col0(Left Semi),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col3 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_107] (rows=144002668 width=135) - predicate:(ws_ship_addr_sk is not null and ws_web_site_sk is not null and ws_ship_date_sk is not null and ws_order_number is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Group By Operator [GBY_44] (rows=174243235 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_31] (rows=174243235 width=135) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_118] (rows=174243235 width=135) - Conds:RS_28._col0=RS_29._col0(Inner),Output:["_col1"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=14398467 width=92) - Output:["_col0"] - Filter Operator [FIL_112] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_25] (rows=14398467 width=92) - default@web_returns,wr,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_23] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_117] (rows=158402938 width=135) - Conds:RS_20._col1=RS_21._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col1 - Select Operator [SEL_16] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_110] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_14] (rows=144002668 width=135) - default@web_sales,ws4,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Select Operator [SEL_19] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_111] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_17] (rows=144002668 width=135) - default@web_sales,ws5,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Group By Operator [GBY_42] (rows=158402938 width=135) - Output:["_col0"],keys:_col0 - Select Operator [SEL_13] (rows=158402938 width=135) - Output:["_col0"] - Filter Operator [FIL_12] (rows=158402938 width=135) - predicate:(_col0 <> _col2) - Merge Join Operator [MERGEJOIN_116] (rows=158402938 width=135) - Conds:RS_9._col1=RS_10._col1(Inner),Output:["_col0","_col1","_col2"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col1 - Select Operator [SEL_5] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_108] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_3] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_109] (rows=144002668 width=135) - predicate:ws_order_number is not null - TableScan [TS_6] (rows=144002668 width=135) - default@web_sales,ws3,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query96.q.out ql/src/test/results/clientpositive/perf/query96.q.out index a6c1f46..ad5bb1d 100644 --- ql/src/test/results/clientpositive/perf/query96.q.out +++ ql/src/test/results/clientpositive/perf/query96.q.out @@ -5,76 +5,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 6 - File Output Operator [FS_29] - Limit [LIM_28] (rows=1 width=8) - Number of rows:100 - Select Operator [SEL_27] (rows=1 width=8) - Output:["_col0"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_26] - Group By Operator [GBY_24] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_45] (rows=766650239 width=88) - Conds:RS_18._col2=RS_19._col0(Inner) - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_42] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - TableScan [TS_9] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_44] (rows=696954748 width=88) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_41] (rows=3600 width=107) - predicate:((hd_dep_count = 5) and hd_demo_sk is not null) - TableScan [TS_6] (rows=7200 width=107) - default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_43] (rows=633595212 width=88) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_39] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=14400 width=471) - Output:["_col0"] - Filter Operator [FIL_40] (rows=14400 width=471) - predicate:((t_hour = 8) and (t_minute >= 30) and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] + Vertex 6 diff --git ql/src/test/results/clientpositive/perf/query97.q.out ql/src/test/results/clientpositive/perf/query97.q.out index d9c994f..24cc0d0 100644 --- ql/src/test/results/clientpositive/perf/query97.q.out +++ ql/src/test/results/clientpositive/perf/query97.q.out @@ -5,91 +5,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 6 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (PARTITION_EDGE) +Vertex 8 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Vertex 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 5 - File Output Operator [FS_38] - Limit [LIM_37] (rows=1 width=24) - Number of rows:100 - Group By Operator [GBY_35] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_34] - Group By Operator [GBY_33] (rows=1 width=24) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col0)","sum(_col1)","sum(_col2)"] - Select Operator [SEL_31] (rows=348477374 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_49] (rows=348477374 width=88) - Conds:RS_28._col0, _col1=RS_29._col0, _col1(Outer),Output:["_col0","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=316797606 width=88) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_11] - PartitionCols:_col0, _col1 - Group By Operator [GBY_10] (rows=633595212 width=88) - Output:["_col0","_col1"],keys:_col2, _col1 - Select Operator [SEL_9] (rows=633595212 width=88) - Output:["_col2","_col1"] - Merge Join Operator [MERGEJOIN_47] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_43] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_44] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_29] - PartitionCols:_col0, _col1 - Group By Operator [GBY_26] (rows=158394413 width=135) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0, _col1 - Group By Operator [GBY_24] (rows=316788826 width=135) - Output:["_col0","_col1"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_48] (rows=316788826 width=135) - Conds:RS_20._col0=RS_21._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_46] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) - TableScan [TS_17] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col0 - Select Operator [SEL_16] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_45] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_14] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk"] + Vertex 5 diff --git ql/src/test/results/clientpositive/perf/query98.q.out ql/src/test/results/clientpositive/perf/query98.q.out index 1bae9be..8a444ee 100644 --- ql/src/test/results/clientpositive/perf/query98.q.out +++ ql/src/test/results/clientpositive/perf/query98.q.out @@ -15,74 +15,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 3 <- Map 8 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Vertex 4 (SORT_PARTITION_EDGE) +Vertex 6 <- Vertex 5 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_27] - Select Operator [SEL_26] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_25] - Select Operator [SEL_23] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - PTF Operator [PTF_22] (rows=348477374 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"_col3"}] - Select Operator [SEL_21] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_20] - PartitionCols:_col3 - Select Operator [SEL_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_18] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_16] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col2)"],keys:_col10, _col9, _col6, _col7, _col8 - Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col10","_col9","_col6","_col7","_col8","_col2"] - Merge Join Operator [MERGEJOIN_37] (rows=696954748 width=88) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col6","_col7","_col8","_col9","_col10"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_35] (rows=231000 width=1436) - predicate:((i_category) IN ('Jewelry', 'Sports', 'Books') and i_item_sk is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc","i_current_price","i_class","i_category"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_36] (rows=633595212 width=88) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_33] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_34] (rows=8116 width=1119) - predicate:(d_date BETWEEN 2001-01-12 AND 2001-02-11 00:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Vertex 6 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index 6602222..6321eab 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -13,30 +13,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 3 <- Union 2 (CONTAINS) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:10 Stage-1 Union 2 - <-Map 1 [CONTAINS] - File Output Operator [FS_7] - Limit [LIM_6] (rows=10/20 width=178) - Number of rows:10 - Select Operator [SEL_1] (rows=500/12 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/13 width=178) - Output:["key","value"] - <-Map 3 [CONTAINS] - File Output Operator [FS_7] - Limit [LIM_6] (rows=10/20 width=178) - Number of rows:10 - Select Operator [SEL_3] (rows=500/12 width=178) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=500/13 width=178) - Output:["key","value"] + <-Vertex 1 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: select key from src PREHOOK: type: QUERY @@ -84,13 +70,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.t"} - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -128,13 +108,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.t"} - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["_col0"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 1 PREHOOK: query: select key from src limit 10 PREHOOK: type: QUERY @@ -232,23 +206,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=205/309 width=95) - Output:["_col0","_col1"],aggregations:["count(KEY._col0)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - PartitionCols:key - Select Operator [SEL_1] (rows=500/500 width=87) - Output:["key"] - TableScan [TS_0] (rows=500/500 width=87) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: select count(*) from (select key, count(key) from src group by key)subq PREHOOK: type: QUERY @@ -381,26 +345,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 3 <- Union 2 (CONTAINS) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 3 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 Union 2 - <-Map 1 [CONTAINS] - File Output Operator [FS_6] - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - Output:["key","value"] - <-Map 3 [CONTAINS] - File Output Operator [FS_6] - Select Operator [SEL_3] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_2] (rows=500/500 width=178) - Output:["key","value"] + <-Vertex 1 [CONTAINS] + <-Vertex 3 [CONTAINS] PREHOOK: query: select count(*) from (select * from src a union all select * from src b)subq PREHOOK: type: QUERY @@ -432,34 +386,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_10] - Select Operator [SEL_9] (rows=1219/1028 width=178) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_15] (rows=1219/1028 width=178) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=87) - default@src,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_14] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_3] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out index 88c68d4..8747e50 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out @@ -41,139 +41,23 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 11 <- Union 12 (CONTAINS) -Map 16 <- Union 12 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 13 <- Union 12 (SIMPLE_EDGE) -Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 12 (CONTAINS) +Vertex 13 <- Union 12 (SORT_PARTITION_EDGE) +Vertex 14 <- Vertex 13 (SORT_PARTITION_EDGE), Vertex 17 (SORT_PARTITION_EDGE) +Vertex 15 <- Union 6 (CONTAINS), Vertex 14 (SORT_PARTITION_EDGE), Vertex 18 (SORT_PARTITION_EDGE) +Vertex 16 <- Union 12 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE), Vertex 9 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 10 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE) +Vertex 8 <- Union 2 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 - File Output Operator [FS_56] - Group By Operator [GBY_54] (rows=28/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1 - Select Operator [SEL_49] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_82] (rows=148/61 width=177) - Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col1","_col2"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_47] - PartitionCols:_col0 - Select Operator [SEL_42] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_78] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_40] (rows=500/500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_81] (rows=61/52 width=177) - Conds:RS_43._col1=RS_44._col1(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col1 - Select Operator [SEL_39] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_37] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Select Operator [SEL_36] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 12 [SIMPLE_EDGE] - <-Map 11 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_27] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_25] (rows=25/25 width=175) - Output:["key","value"] - <-Map 16 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_30] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_28] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_53] - PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_80] (rows=148/61 width=177) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=87) - Output:["_col0"] - Filter Operator [FIL_74] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_15] (rows=500/500 width=87) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_79] (rows=61/52 width=177) - Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_73] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_12] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_2] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_71] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25/25 width=175) - Output:["key","value"] - <-Map 8 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_72] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500/500 width=178) - Output:["key","value"] + Vertex 7 PREHOOK: query: SELECT x.key, y.value FROM src1 x JOIN src y ON (x.key = y.key) @@ -234,264 +118,35 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Union 2 (CONTAINS) -Map 10 <- Union 2 (CONTAINS) -Map 13 <- Union 14 (CONTAINS) -Map 20 <- Union 14 (CONTAINS) -Map 21 <- Union 16 (CONTAINS) -Map 24 <- Union 25 (CONTAINS) -Map 33 <- Union 25 (CONTAINS) -Map 34 <- Union 27 (CONTAINS) -Map 35 <- Union 29 (CONTAINS) -Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) -Reducer 17 <- Union 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS) -Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 30 <- Union 29 (SIMPLE_EDGE) -Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) -Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) -Reducer 9 <- Union 8 (SIMPLE_EDGE) +Vertex 1 <- Union 2 (CONTAINS) +Vertex 10 <- Union 2 (CONTAINS) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 6 (CONTAINS), Vertex 18 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 14 (CONTAINS) +Vertex 21 <- Union 16 (CONTAINS) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 8 (CONTAINS), Vertex 31 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 33 <- Union 25 (CONTAINS) +Vertex 34 <- Union 27 (CONTAINS) +Vertex 35 <- Union 29 (CONTAINS) +Vertex 4 <- Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 6 (CONTAINS), Vertex 12 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 7 <- Union 6 (SORT_PARTITION_EDGE), Union 8 (CONTAINS) +Vertex 9 <- Union 8 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 - File Output Operator [FS_114] - Group By Operator [GBY_112] (rows=872/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 8 [SIMPLE_EDGE] - <-Reducer 32 [CONTAINS] - Reduce Output Operator [RS_111] - PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=434/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_162] (rows=434/61 width=177) - Conds:RS_104._col2=RS_105._col0(Inner),Output:["_col2","_col5"] - <-Map 37 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_100] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_98] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_161] (rows=178/52 width=86) - Conds:RS_101._col1=RS_102._col1(Inner),Output:["_col2"] - <-Map 36 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col1 - Select Operator [SEL_97] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_95] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col1 - Select Operator [SEL_94] (rows=1525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_93] (rows=1525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 29 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] - Reduce Output Operator [RS_92] - PartitionCols:_col1, _col0 - Select Operator [SEL_88] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_86] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 28 [CONTAINS] - Reduce Output Operator [RS_92] - PartitionCols:_col1, _col0 - Select Operator [SEL_85] (rows=1025/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_84] (rows=1025/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 27 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] - Reduce Output Operator [RS_83] - PartitionCols:_col1, _col0 - Select Operator [SEL_79] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_77] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 26 [CONTAINS] - Reduce Output Operator [RS_83] - PartitionCols:_col1, _col0 - Select Operator [SEL_76] (rows=525/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_75] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 25 [SIMPLE_EDGE] - <-Map 24 [CONTAINS] - Reduce Output Operator [RS_74] - PartitionCols:_col1, _col0 - Select Operator [SEL_67] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_151] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_65] (rows=25/25 width=175) - Output:["key","value"] - <-Map 33 [CONTAINS] - Reduce Output Operator [RS_74] - PartitionCols:_col1, _col0 - Select Operator [SEL_70] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_68] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 7 [CONTAINS] - Reduce Output Operator [RS_111] - PartitionCols:_col0, _col1 - Group By Operator [GBY_63] (rows=438/15 width=177) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 6 [SIMPLE_EDGE] - <-Reducer 19 [CONTAINS] - Reduce Output Operator [RS_62] - PartitionCols:_col0, _col1 - Select Operator [SEL_58] (rows=290/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_160] (rows=290/61 width=177) - Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_150] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_49] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_159] (rows=119/52 width=86) - Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col1 - Select Operator [SEL_48] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_149] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_46] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col1 - Select Operator [SEL_45] (rows=1025/319 width=178) - Output:["_col1"] - Group By Operator [GBY_44] (rows=1025/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 16 [SIMPLE_EDGE] - <-Map 21 [CONTAINS] - Reduce Output Operator [RS_43] - PartitionCols:_col1, _col0 - Select Operator [SEL_39] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_148] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_37] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_43] - PartitionCols:_col1, _col0 - Select Operator [SEL_36] (rows=525/319 width=178) - Output:["_col0","_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_27] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_146] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_25] (rows=25/25 width=175) - Output:["key","value"] - <-Map 20 [CONTAINS] - Reduce Output Operator [RS_34] - PartitionCols:_col1, _col0 - Select Operator [SEL_30] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_147] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_28] (rows=500/500 width=178) - Output:["key","value"] - <-Reducer 5 [CONTAINS] - Reduce Output Operator [RS_62] - PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=148/61 width=177) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_158] (rows=148/61 width=177) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_15] (rows=500/500 width=178) - default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_157] (rows=61/52 width=86) - Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_144] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - TableScan [TS_12] (rows=25/25 width=175) - default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_2] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_142] (rows=25/25 width=175) - predicate:value is not null - TableScan [TS_0] (rows=25/25 width=175) - Output:["key","value"] - <-Map 10 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col1, _col0 - Select Operator [SEL_5] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_143] (rows=500/500 width=178) - predicate:value is not null - TableScan [TS_3] (rows=500/500 width=178) - Output:["key","value"] + Vertex 9 PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE PREHOOK: type: CREATETABLE @@ -647,22 +302,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value PREHOOK: type: QUERY @@ -683,42 +323,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292/1166 width=18) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242/242 width=18) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242/242 width=18) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242/242 width=18) - predicate:value is not null - TableScan [TS_6] (rows=242/242 width=18) - default@tab,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key PREHOOK: type: QUERY @@ -746,22 +357,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_14] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Vertex 1 PREHOOK: query: select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key join tab2 s2 on s1.value=s2.value PREHOOK: type: QUERY @@ -786,42 +382,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_16] - Merge Join Operator [MERGEJOIN_27] (rows=292/1166 width=18) - Conds:RS_12._col1=RS_13._col1(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_25] (rows=266/480 width=18) - Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] - <-Select Operator [SEL_5] (rows=242/242 width=18) - Output:["_col0"] - Filter Operator [FIL_23] (rows=242/242 width=18) - predicate:key is not null - TableScan [TS_3] (rows=242/242 width=18) - default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Select Operator [SEL_2] (rows=242/242 width=18) - Output:["_col0","_col1"] - Filter Operator [FIL_22] (rows=242/242 width=18) - predicate:(key is not null and value is not null) - TableScan [TS_0] (rows=242/242 width=18) - default@tab,s1,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col1 - Select Operator [SEL_8] (rows=242/242 width=18) - Output:["_col1"] - Filter Operator [FIL_24] (rows=242/242 width=18) - predicate:value is not null - TableScan [TS_6] (rows=242/242 width=18) - default@tab2,s2,Tbl:COMPLETE,Col:NONE,Output:["value"] + Vertex 2 PREHOOK: query: select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key UNION ALL @@ -1126,6 +693,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage +<<<<<<< HEAD Map 1 <- Union 2 (CONTAINS) Map 13 <- Union 10 (CONTAINS) Map 14 <- Union 10 (CONTAINS) @@ -1141,6 +709,23 @@ Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Union 22 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) Reducer 4 <- Map 8 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) +======= +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Union 10 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 15 (SORT_PARTITION_EDGE) +Vertex 12 <- Union 10 (CONTAINS) +Vertex 13 <- Union 10 (CONTAINS) +Vertex 15 <- Vertex 14 (SORT_PARTITION_EDGE), Vertex 16 (SORT_PARTITION_EDGE) +Vertex 17 <- Union 18 (CONTAINS) +Vertex 19 <- Union 18 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 24 (SORT_PARTITION_EDGE) +Vertex 20 <- Union 18 (CONTAINS) +Vertex 21 <- Union 18 (CONTAINS) +Vertex 22 <- Union 18 (CONTAINS) +Vertex 24 <- Vertex 23 (SORT_PARTITION_EDGE), Vertex 25 (SORT_PARTITION_EDGE) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE), Union 4 (CONTAINS), Vertex 7 (SORT_PARTITION_EDGE) +Vertex 5 <- Union 2 (CONTAINS) +Vertex 7 <- Vertex 6 (SORT_PARTITION_EDGE), Vertex 8 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 10 (CONTAINS) +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator @@ -1150,6 +735,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 +<<<<<<< HEAD Union 5 <-Reducer 12 [CONTAINS] File Output Operator [FS_79] @@ -1343,6 +929,12 @@ Stage-5 File Output Operator [FS_83] table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_20] +======= + Union 4 + <-Vertex 11 [CONTAINS] + <-Vertex 19 [CONTAINS] + <-Vertex 3 [CONTAINS] +>>>>>>> renamed edges Stage-6 Stats-Aggr Operator Stage-1 @@ -1431,6 +1023,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage +<<<<<<< HEAD Map 1 <- Union 2 (CONTAINS) Map 10 <- Union 2 (CONTAINS) Map 13 <- Union 14 (CONTAINS) @@ -1454,6 +1047,31 @@ Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 8 (SIMPLE_EDGE) +======= +Vertex 1 <- Union 2 (CONTAINS) +Vertex 11 <- Vertex 10 (SORT_PARTITION_EDGE), Vertex 12 (SORT_PARTITION_EDGE) +Vertex 13 <- Union 14 (CONTAINS) +Vertex 15 <- Union 14 (SORT_PARTITION_EDGE), Union 16 (CONTAINS) +Vertex 17 <- Union 16 (SORT_PARTITION_EDGE) +Vertex 18 <- Union 5 (CONTAINS), Vertex 17 (SORT_PARTITION_EDGE), Vertex 22 (SORT_PARTITION_EDGE) +Vertex 19 <- Union 14 (CONTAINS) +Vertex 20 <- Union 16 (CONTAINS) +Vertex 22 <- Vertex 21 (SORT_PARTITION_EDGE), Vertex 23 (SORT_PARTITION_EDGE) +Vertex 24 <- Union 25 (CONTAINS) +Vertex 26 <- Union 25 (SORT_PARTITION_EDGE), Union 27 (CONTAINS) +Vertex 28 <- Union 27 (SORT_PARTITION_EDGE), Union 29 (CONTAINS) +Vertex 3 <- Union 2 (SORT_PARTITION_EDGE) +Vertex 30 <- Union 29 (SORT_PARTITION_EDGE) +Vertex 31 <- Union 7 (CONTAINS), Vertex 30 (SORT_PARTITION_EDGE), Vertex 36 (SORT_PARTITION_EDGE) +Vertex 32 <- Union 25 (CONTAINS) +Vertex 33 <- Union 27 (CONTAINS) +Vertex 34 <- Union 29 (CONTAINS) +Vertex 36 <- Vertex 35 (SORT_PARTITION_EDGE), Vertex 37 (SORT_PARTITION_EDGE) +Vertex 4 <- Union 5 (CONTAINS), Vertex 11 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) +Vertex 6 <- Union 5 (SORT_PARTITION_EDGE), Union 7 (CONTAINS) +Vertex 8 <- Union 7 (SORT_PARTITION_EDGE) +Vertex 9 <- Union 2 (CONTAINS) +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator @@ -1463,6 +1081,7 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 +<<<<<<< HEAD Reducer 9 File Output Operator [FS_115] table:{"name:":"default.a"} @@ -1700,6 +1319,9 @@ Stage-5 File Output Operator [FS_119] table:{"name:":"default.c"} Please refer to the previous Group By Operator [GBY_112] +======= + Vertex 8 +>>>>>>> renamed edges Stage-6 Stats-Aggr Operator Stage-1 @@ -1777,6 +1399,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 5 File Output Operator [FS_18] table:{"name:":"default.dest1"} @@ -1812,6 +1435,9 @@ Stage-4 Group By Operator [GBY_21] (rows=501/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_12] +======= + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 @@ -1846,33 +1472,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO due to missing feature [Unique_join]. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=594/4122 width=260) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_9] (rows=594/4122 width=260) - Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - TableScan [TS_0] (rows=500/500 width=87) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_4] - PartitionCols:key - TableScan [TS_1] (rows=25/25 width=86) - default@src1,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - TableScan [TS_2] (rows=2000/2000 width=87) - default@srcpart,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: SELECT TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) @@ -1905,32 +1511,13 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_9] - Transform Operator [SCR_8] (rows=1219/1028 width=178) - command:cat - Merge Join Operator [MERGEJOIN_14] (rows=1219/1028 width=178) - Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - Filter Operator [FIL_12] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_1] (rows=500/500 width=87) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Vertex 2 PREHOOK: query: FROM ( select key, value from ( @@ -2005,6 +1592,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 4 File Output Operator [FS_17] table:{"name:":"default.dest1"} @@ -2052,6 +1640,10 @@ Stage-4 Group By Operator [GBY_20] (rows=1001/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] +======= + Vertex 4 + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 @@ -2119,6 +1711,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 +<<<<<<< HEAD Reducer 4 File Output Operator [FS_14] table:{"name:":"default.dest1"} @@ -2156,6 +1749,10 @@ Stage-4 Group By Operator [GBY_17] (rows=501/310 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] +======= + Vertex 4 + Vertex 5 +>>>>>>> renamed edges Stage-5 Stats-Aggr Operator Stage-1 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 9f1a401..7c5df3c 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -227,9 +227,7 @@ POSTHOOK: type: QUERY Stage-2 Stats-Aggr Operator Stage-0 - Map 1 - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:COMPLETE + Vertex 1 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -345,13 +343,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.src_autho_test"} - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -599,24 +591,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 2 - File Output Operator [FS_5] - Limit [LIM_4] (rows=5/5 width=178) - Number of rows:5 - Select Operator [SEL_3] (rows=500/5 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc PREHOOK: type: CREATETABLE @@ -660,15 +641,7 @@ Stage-3 Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) Conditional Operator Stage-1 - Map 1 - File Output Operator [FS_3] - table:{"name:":"default.orc_merge5"} - Select Operator [SEL_2] (rows=306/3 width=268) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_4] (rows=306/3 width=268) - predicate:(userid <= 13) - TableScan [TS_0] (rows=919/15000 width=268) - default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] + Vertex 1 Stage-4(CONDITIONAL) File Merge Please refer to the previous Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) diff --git ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out index 14535f6..715bcbf 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out @@ -35,39 +35,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 4 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=2166/10 width=620) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Merge Join Operator [MERGEJOIN_17] (rows=2166/10 width=620) - Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365/10 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365/10 width=251) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=251) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 4 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1019/10 width=251) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1019/10 width=251) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=251) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -222,45 +197,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 5 (SORT_PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_15] - Select Operator [SEL_14] (rows=615/5 width=12) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_13] - Group By Operator [GBY_11] (rows=615/5 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_20] (rows=2166/10 width=4) - Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365/10 width=5) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=1365/10 width=5) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=5) - default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"] - <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1019/10 width=8) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1019/10 width=8) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=8) - default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 @@ -324,39 +269,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 4 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_12] - Select Operator [SEL_11] (rows=1501/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - Map Join Operator [MAPJOIN_17] (rows=1501/10 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"] - <-Map 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col2 - Select Operator [SEL_5] (rows=1365/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_16] (rows=1365/10 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col2 - Select Operator [SEL_2] (rows=1365/10 width=215) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_15] (rows=1365/10 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] + Vertex 3 PREHOOK: query: select * @@ -511,45 +431,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (PARTITION_EDGE), Vertex 5 (PARTITION_EDGE) +Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) +Vertex 4 <- Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_15] - Select Operator [SEL_14] (rows=750/5 width=215) - Output:["_col0","_col1"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_13] - Group By Operator [GBY_11] (rows=750/5 width=215) - Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0 - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_10] - PartitionCols:_col0 - Map Join Operator [MAPJOIN_20] (rows=1501/10 width=215) - Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=1365/10 width=215) - Output:["_col0"] - Filter Operator [FIL_19] (rows=1365/10 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) - TableScan [TS_3] (rows=12288/12288 width=215) - default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_6] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=1365/10 width=215) - Output:["_col0","_col1"] - Filter Operator [FIL_18] (rows=1365/10 width=215) - predicate:cint BETWEEN 1000000 AND 3000000 - TableScan [TS_0] (rows=12288/12288 width=215) - default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"] + Vertex 4 PREHOOK: query: select a.csmallint, count(*) c1 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out index ee9affb..e4a623f 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out @@ -29,9 +29,7 @@ POSTHOOK: type: QUERY Stage-2 Stats-Aggr Operator Stage-0 - Map 1 - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:COMPLETE + Vertex 1 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -279,7 +277,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -289,20 +287,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 - File Output Operator [FS_8] - table:{"name:":"default.acid_uami"} - Select Operator [SEL_4] (rows=8/2 width=302) - Output:["_col0","_col1","_col2","_col3"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=8/2 width=302) - Output:["_col0","_col1","_col3"] - Filter Operator [FIL_9] (rows=8/2 width=226) - predicate:((de = 109.23) or (de = 119.23)) - TableScan [TS_0] (rows=8/4 width=226) - default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["i","de","vc"] + Vertex 2 PREHOOK: query: select * from acid_uami order by de PREHOOK: type: QUERY @@ -392,7 +377,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze delete from acid_dot where cint < -1070551679 POSTHOOK: type: QUERY Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-3 Stats-Aggr Operator @@ -402,20 +387,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Reducer 2 - File Output Operator [FS_7] - table:{"name:":"default.acid_dot"} - Select Operator [SEL_4] (rows=31436/8 width=4) - Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=31436/8 width=4) - Output:["_col0"] - Filter Operator [FIL_8] (rows=31436/8 width=4) - predicate:(cint < -1070551679) - TableScan [TS_0] (rows=94309/12288 width=4) - default@acid_dot,acid_dot, ACID table,Tbl:COMPLETE,Col:NONE,Output:["cint"] + Vertex 2 PREHOOK: query: select count(*) from acid_dot PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/tez/explainuser_3.q.out ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 17c9ec3..dd9e607 100644 --- ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -271,13 +271,7 @@ Stage-3 Stage-2 Dependency Collection{} Stage-1 - Map 1 - File Output Operator [FS_2] - table:{"name:":"default.src_autho_test"} - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 1 Stage-0 Move Operator Please refer to the previous Stage-1 @@ -451,24 +445,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 2 vectorized - File Output Operator [FS_8] - Limit [LIM_7] (rows=5 width=178) - Number of rows:5 - Select Operator [SEL_6] (rows=500 width=178) - Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Vertex 2 PREHOOK: query: create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc PREHOOK: type: CREATETABLE @@ -504,15 +487,7 @@ Stage-3 Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) Conditional Operator Stage-1 - Map 1 vectorized - File Output Operator [FS_10] - table:{"name:":"default.orc_merge5"} - Select Operator [SEL_9] (rows=306 width=268) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_8] (rows=306 width=268) - predicate:(userid <= 13) - TableScan [TS_0] (rows=919 width=268) - default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] + Vertex 1 Stage-4(CONDITIONAL) File Merge Please refer to the previous Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) diff --git ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out index f3a5dc3..01c7379 100644 --- ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -55,7 +55,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -66,7 +66,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -83,20 +97,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -171,7 +171,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -183,7 +183,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -200,20 +214,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -286,7 +286,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -297,7 +297,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -314,20 +328,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -398,7 +398,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 19518 Data size: 156144 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -410,7 +410,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: cd @@ -427,20 +441,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -508,7 +508,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -519,21 +519,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: cd - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -547,6 +533,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: cd + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -614,7 +614,7 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 26150 Data size: 209200 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -626,21 +626,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: cd - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -654,6 +640,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 + Map Operator Tree: + TableScan + alias: cd + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -756,7 +756,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -767,7 +767,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -784,20 +798,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -872,7 +872,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -884,7 +884,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -901,20 +915,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -989,7 +989,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1000,7 +1000,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1017,20 +1031,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1105,7 +1105,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: - 1 Map 3 + 1 Vertex 3 Statistics: Num rows: 577 Data size: 18341 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator @@ -1117,7 +1117,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Map 3 + Vertex 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 3 Map Operator Tree: TableScan alias: p2 @@ -1134,20 +1148,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 525 Data size: 16674 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1206,7 +1206,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1226,10 +1226,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1249,7 +1249,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1263,7 +1263,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1287,7 +1287,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1402,7 +1402,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1422,10 +1422,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Vertex 1 <- Vertex 2 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: l @@ -1445,7 +1445,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Vertex 2 Statistics: Num rows: 37748736 Data size: 13339877376 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) @@ -1459,7 +1459,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: r @@ -1483,7 +1483,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Vertex 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out index 6f5a3a9..cd42d3a 100644 --- ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out +++ ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_2.q.out @@ -52,7 +52,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -69,8 +69,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -81,20 +81,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -108,6 +95,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -176,7 +176,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -193,8 +193,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -206,20 +206,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: y - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Vertex 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -233,6 +220,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -312,7 +312,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -331,9 +331,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -344,7 +344,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -357,7 +371,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -370,20 +384,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -456,7 +456,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -475,9 +475,9 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) input vertices: - 0 Map 1 - 2 Map 4 - 3 Map 5 + 0 Vertex 1 + 2 Vertex 4 + 3 Vertex 5 Statistics: Num rows: 5803 Data size: 46424 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -489,7 +489,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: w @@ -502,7 +516,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: y @@ -515,20 +529,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -619,7 +619,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -632,7 +632,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -649,8 +649,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -661,7 +661,38 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 6 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -674,7 +705,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -687,7 +718,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -704,8 +735,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -716,38 +747,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -764,8 +764,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -854,7 +852,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Map 10 + Vertex 10 Map Operator Tree: TableScan alias: y @@ -867,7 +865,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z @@ -884,8 +882,8 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) input vertices: - 0 Map 1 - 2 Map 6 + 0 Vertex 1 + 2 Vertex 6 Statistics: Num rows: 594 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -897,7 +895,38 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 6 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Vertex 5 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: bigint) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 6 Map Operator Tree: TableScan alias: y @@ -910,7 +939,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map 7 + Vertex 7 Map Operator Tree: TableScan alias: x @@ -923,7 +952,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE - Map 8 + Vertex 8 Map Operator Tree: TableScan alias: z @@ -940,8 +969,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 0 Map 7 - 2 Map 10 + 0 Vertex 7 + 2 Vertex 10 Statistics: Num rows: 545 Data size: 4360 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -953,38 +982,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 9 + Vertex 9 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1001,8 +999,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union 4 - Vertex: Union 4 Stage: Stage-0 Fetch Operator @@ -1100,7 +1096,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: value (type: string) - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1118,8 +1114,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: @@ -1130,8 +1126,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() @@ -1142,7 +1138,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1155,7 +1165,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1168,7 +1178,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1181,20 +1191,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1284,7 +1280,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 350 Basic stats: COMPLETE Column stats: COMPLETE value expressions: value (type: string) - Map 2 + Vertex 2 Map Operator Tree: TableScan alias: z1 @@ -1302,8 +1298,8 @@ STAGE PLANS: 2 key (type: string) outputColumnNames: _col1 input vertices: - 0 Map 1 - 2 Map 4 + 0 Vertex 1 + 2 Vertex 4 Statistics: Num rows: 46 Data size: 4094 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Map Join Operator @@ -1315,8 +1311,8 @@ STAGE PLANS: 1 value (type: string) 2 value (type: string) input vertices: - 1 Map 5 - 2 Map 6 + 1 Vertex 5 + 2 Vertex 6 Statistics: Num rows: 981 Data size: 7848 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator @@ -1328,7 +1324,21 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Map 4 + Vertex 3 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Vertex 4 Map Operator Tree: TableScan alias: y1 @@ -1341,7 +1351,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE - Map 5 + Vertex 5 Map Operator Tree: TableScan alias: z2 @@ -1354,7 +1364,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 222 Data size: 39516 Basic stats: COMPLETE Column stats: COMPLETE - Map 6 + Vertex 6 Map Operator Tree: TableScan alias: y2 @@ -1367,20 +1377,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out index 8006a90..4165c57 100644 --- ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out +++ ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out @@ -102,36 +102,13 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE), Vertex 3 (SORT_PARTITION_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_21] (rows=2 width=431) - Conds:RS_23._col2=RS_28._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_23] - PartitionCols:_col2 - Select Operator [SEL_22] (rows=2 width=134) - Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=2 width=236) - default@char_tbl1,c1,Tbl:COMPLETE,Col:PARTIAL,Output:["name","age"] - Dynamic Partitioning Event Operator [EVENT_26] (rows=1 width=134) - Group By Operator [GBY_25] (rows=1 width=134) - Output:["_col0"],keys:_col0 - Select Operator [SEL_24] (rows=2 width=134) - Output:["_col0"] - Please refer to the previous Select Operator [SEL_22] - <-Map 3 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_28] - PartitionCols:_col2 - Select Operator [SEL_27] (rows=2 width=89) - Output:["_col0","_col1","_col2"] - TableScan [TS_3] (rows=2 width=190) - default@char_tbl2,c2,Tbl:COMPLETE,Col:PARTIAL,Output:["name","age"] + Vertex 2 PREHOOK: query: select c1.name, c1.age, c1.gpa, c2.name, c2.age, c2.gpa from char_tbl1 c1 join char_tbl2 c2 on (c1.gpa = c2.gpa) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out index 4535e66..624ace1 100644 --- ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out +++ ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out @@ -40,10 +40,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: non_string_part @@ -62,7 +62,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: tinyint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -119,10 +119,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: non_string_part @@ -140,7 +140,7 @@ STAGE PLANS: Statistics: Num rows: 1024 Data size: 121205 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/tez/vectorization_div0.q.out ql/src/test/results/clientpositive/tez/vectorization_div0.q.out index 3c017e6..a4431b0 100644 --- ql/src/test/results/clientpositive/tez/vectorization_div0.q.out +++ ql/src/test/results/clientpositive/tez/vectorization_div0.q.out @@ -146,10 +146,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -168,7 +168,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col2 (type: decimal(22,21)) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -319,10 +319,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -341,7 +341,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.1 value expressions: _col2 (type: double), _col4 (type: double), _col5 (type: double) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator diff --git ql/src/test/results/clientpositive/tez/vectorization_limit.q.out ql/src/test/results/clientpositive/tez/vectorization_limit.q.out index dfb0102..8d50a8c 100644 --- ql/src/test/results/clientpositive/tez/vectorization_limit.q.out +++ ql/src/test/results/clientpositive/tez/vectorization_limit.q.out @@ -53,10 +53,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -75,7 +75,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 value expressions: _col2 (type: smallint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Select Operator @@ -142,10 +142,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -168,7 +168,7 @@ STAGE PLANS: TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: struct) Execution mode: vectorized - Reducer 2 + Vertex 2 Reduce Operator Tree: Group By Operator aggregations: avg(VALUE._col0) @@ -236,10 +236,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -260,7 +260,7 @@ STAGE PLANS: Statistics: Num rows: 95 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -328,10 +328,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -351,7 +351,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: tinyint) Statistics: Num rows: 6144 Data size: 55052 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -449,11 +449,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Vertex 2 <- Vertex 1 (SORT_PARTITION_EDGE) + Vertex 3 <- Vertex 2 (SORT_PARTITION_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Vertex 1 Map Operator Tree: TableScan alias: alltypesorc @@ -478,7 +478,7 @@ STAGE PLANS: Statistics: Num rows: 3185 Data size: 44512 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized - Reducer 2 + Vertex 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -492,7 +492,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 3185 Data size: 44512 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 - Reducer 3 + Vertex 3 Execution mode: vectorized Reduce Operator Tree: Select Operator